mirror of
https://github.com/moses-smt/mosesdecoder.git
synced 2024-12-27 05:55:02 +03:00
995 lines
32 KiB
Perl
Executable File
995 lines
32 KiB
Perl
Executable File
#!/usr/bin/env perl
|
|
|
|
use strict;
|
|
#eddie specific
|
|
use lib "/exports/informatics/inf_iccs_smt/perl/lib/perl5/site_perl";
|
|
use Config::Simple;
|
|
use File::Basename;
|
|
use Getopt::Long "GetOptions";
|
|
|
|
my ($config_file,$execute,$continue);
|
|
die("training-expt.perl -config config-file [-exec]")
|
|
unless &GetOptions('config=s' => \$config_file,
|
|
'cont=s' => \$continue,
|
|
'exec' => \$execute);
|
|
|
|
my $config = new Config::Simple($config_file) ||
|
|
die "Error: unable to read config file \"$config_file\"";
|
|
|
|
#substitution
|
|
foreach my $key ($config->param) {
|
|
my $value = $config->param($key);
|
|
while ($value =~ m/(.*?)\$\{(.*?)\}(.*)/) {
|
|
my $sub = $config->param($2);
|
|
if (! $sub) {
|
|
#try in this scope
|
|
my $scope = (split /\./, $key)[0];
|
|
$sub = $config->param($scope . "." . $2);
|
|
}
|
|
if (! $sub) {
|
|
#then general
|
|
$sub = $config->param("general." . $2);
|
|
}
|
|
$value = $1 . $sub . $3;
|
|
}
|
|
print STDERR "$key => "; print STDERR $value; print STDERR "\n";
|
|
$config->param($key,$value);
|
|
}
|
|
|
|
#check if we're using sge
|
|
my $have_sge = 0;
|
|
if (`which qsub 2>/dev/null`) {
|
|
print "Using sge for job control.\n";
|
|
$have_sge = 1;
|
|
} else {
|
|
print "No sge detected.\n";
|
|
}
|
|
|
|
#required global parameters
|
|
my $name = ¶m_required("general.name");
|
|
|
|
#optional globals
|
|
my $queue = ¶m("general.queue", "inf_iccs_smt");
|
|
my $mpienv = ¶m("general.mpienv", "openmpi_smp8_mark2");
|
|
my $vmem = ¶m("general.vmem", "6");
|
|
|
|
#wait for bleu files to appear in experiment folder if running as part of experiment.perl
|
|
my $wait_for_bleu = ¶m("general.wait-for-bleu", 0);
|
|
|
|
#job control
|
|
my $jackknife = ¶m("general.jackknife", 0);
|
|
my $working_dir = ¶m("general.working-dir");
|
|
my $general_decoder_settings = ¶m("general.decoder-settings", "");
|
|
system("mkdir -p $working_dir") == 0 or die "Error: unable to create directory \"$working_dir\"";
|
|
my $train_script = "$name-train";
|
|
my $job_name = "$name-t";
|
|
my $hours = ¶m("train.hours",48);
|
|
|
|
# check if we are tuning a meta feature
|
|
my $tuneMetaFeature = ¶m("general.tune-meta-feature", 0);
|
|
print STDERR "Tuning meta feature.. \n" if $tuneMetaFeature;
|
|
|
|
# Check if a weight file with start weights was given
|
|
my $start_weight_file = ¶m("start.weightfile");
|
|
|
|
#required training parameters
|
|
my $singleRef = 1;
|
|
my ($moses_ini_file, $input_file, $reference_files);
|
|
my (@moses_ini_files_folds, @input_files_folds, @reference_files_folds);
|
|
if ($jackknife) {
|
|
my $array_ref = ¶m_required("train.moses-ini-files-folds");
|
|
@moses_ini_files_folds= @$array_ref;
|
|
foreach my $ini (@moses_ini_files_folds) {
|
|
&check_exists ("moses ini file", $ini);
|
|
}
|
|
$array_ref = ¶m_required("train.input-files-folds");
|
|
@input_files_folds = @$array_ref;
|
|
foreach my $in (@input_files_folds) {
|
|
&check_exists ("train input file", $in);
|
|
}
|
|
$array_ref = ¶m_required("train.reference-files-folds");
|
|
@reference_files_folds = @$array_ref;
|
|
foreach my $ref (@reference_files_folds) {
|
|
&check_exists ("train reference file", $ref);
|
|
}
|
|
}
|
|
else {
|
|
$moses_ini_file = ¶m_required("train.moses-ini-file");
|
|
&check_exists ("moses ini file", $moses_ini_file);
|
|
$input_file = ¶m_required("train.input-file");
|
|
&check_exists ("train input file", $input_file);
|
|
$reference_files = ¶m_required("train.reference-files");
|
|
if (&check_exists_noThrow ("ref files", $reference_files) != 0) {
|
|
for my $ref (glob $reference_files . "*") {
|
|
&check_exists ("ref files", $ref);
|
|
}
|
|
$singleRef = 0;
|
|
}
|
|
}
|
|
|
|
# check if we want to continue an interrupted experiment
|
|
my $continue_expt = ¶m("general.continue-expt", 0); # number of experiment to continue
|
|
my $continue_epoch = 0;
|
|
if ($continue_expt > 0) {
|
|
die "ERROR: Continuing an experiment is not defined for tuning meta features.. \n\n" if ($tuneMetaFeature);
|
|
$continue_epoch = ¶m_required("general.continue-epoch", 0);
|
|
my $continue_weights = ¶m_required("general.continue-weights", 0);
|
|
open(CONT_WEIGHTS, $continue_weights);
|
|
my ($wp_weight, @pm_weights, $lm_weight, $lm2_weight, $d_weight, @lr_weights, %extra_weights);
|
|
my $num_core_weights = 0;
|
|
my $num_extra_weights = 0;
|
|
while(<CONT_WEIGHTS>) {
|
|
chomp;
|
|
my ($name,$value) = split;
|
|
next if ($name =~ /^!Unknown/);
|
|
next if ($name =~ /^BleuScore/);
|
|
next if ($name eq "DEFAULT_");
|
|
if ($name eq "WordPenalty") {
|
|
$wp_weight = $value;
|
|
$num_core_weights += 1;
|
|
} elsif ($name =~ /^PhraseModel/) {
|
|
push @pm_weights,$value;
|
|
$num_core_weights += 1;
|
|
} elsif ($name =~ /^LM\:2/) {
|
|
$lm2_weight = $value;
|
|
$num_core_weights += 1;
|
|
}
|
|
elsif ($name =~ /^LM/) {
|
|
$lm_weight = $value;
|
|
$num_core_weights += 1;
|
|
} elsif ($name eq "Distortion") {
|
|
$d_weight = $value;
|
|
$num_core_weights += 1;
|
|
} elsif ($name =~ /^LexicalReordering/) {
|
|
push @lr_weights,$value;
|
|
$num_core_weights += 1;
|
|
} else {
|
|
$extra_weights{$name} = $value;
|
|
$num_extra_weights += 1;
|
|
}
|
|
}
|
|
close CONT_WEIGHTS;
|
|
print STDERR "num core weights to continue: $num_core_weights\n";
|
|
print STDERR "num extra weights to continue: $num_extra_weights\n";
|
|
|
|
# write sparse weights to separate file
|
|
my $sparse_weights = $working_dir."/sparseWeights.expt".$continue_expt;
|
|
if ($num_extra_weights > 0) {
|
|
open(SPARSE, ">$sparse_weights");
|
|
foreach my $name (sort keys %extra_weights) {
|
|
next if ($name eq "core");
|
|
next if ($name eq "DEFAULT_");
|
|
|
|
# write only non-zero feature weights to file
|
|
if ($extra_weights{$name}) {
|
|
print SPARSE "$name $extra_weights{$name}\n";
|
|
}
|
|
}
|
|
close SPARSE;
|
|
}
|
|
|
|
# write new ini files with these weights
|
|
if ($jackknife) {
|
|
my @new_ini_files;
|
|
for (my $i=0; $i<=$#moses_ini_files_folds; $i++) {
|
|
my $ini_continue = $moses_ini_files_folds[$i].".continue".$continue_expt;
|
|
open(OLDINI, $moses_ini_files_folds[$i]);
|
|
open(NEWINI, ">$ini_continue");
|
|
while(<OLDINI>) {
|
|
if (/weight-l/) {
|
|
print NEWINI "[weight-l]\n";
|
|
print NEWINI $lm_weight;
|
|
print NEWINI "\n";
|
|
|
|
if (defined $lm2_weight) {
|
|
readline(OLDINI);
|
|
print NEWINI $lm2_weight;
|
|
print NEWINI "\n";
|
|
}
|
|
|
|
readline(OLDINI);
|
|
} elsif (/weight-t/) {
|
|
print NEWINI "[weight-t]\n";
|
|
foreach my $pm_weight (@pm_weights) {
|
|
print NEWINI $pm_weight;
|
|
print NEWINI "\n";
|
|
readline(OLDINI);
|
|
}
|
|
} elsif (/weight-d/) {
|
|
print NEWINI "[weight-d]\n";
|
|
print NEWINI $d_weight;
|
|
print NEWINI "\n";
|
|
readline(OLDINI);
|
|
foreach my $lr_weight (@lr_weights) {
|
|
print NEWINI $lr_weight;
|
|
print NEWINI "\n";
|
|
readline(OLDINI);
|
|
}
|
|
} elsif (/weight-w/) {
|
|
print NEWINI "[weight-w]\n";
|
|
print NEWINI $wp_weight;
|
|
print NEWINI "\n";
|
|
readline(OLDINI);
|
|
} else {
|
|
print NEWINI;
|
|
}
|
|
}
|
|
if ($num_extra_weights > 0) {
|
|
print NEWINI "\n[weight-file]\n$sparse_weights\n";
|
|
}
|
|
close OLDINI;
|
|
close NEWINI;
|
|
|
|
print STDERR "new ini file: ".$ini_continue."\n";
|
|
$moses_ini_files_folds[$i] = $ini_continue;
|
|
}
|
|
}
|
|
else {
|
|
my $ini_continue = $moses_ini_file.".continue".$continue_expt;
|
|
open(OLDINI, $moses_ini_file);
|
|
open(NEWINI, ">$ini_continue");
|
|
while(<OLDINI>) {
|
|
if (/weight-l/) {
|
|
print NEWINI "[weight-l]\n";
|
|
print NEWINI $lm_weight;
|
|
print NEWINI "\n";
|
|
|
|
if (defined $lm2_weight) {
|
|
readline(OLDINI);
|
|
print NEWINI $lm2_weight;
|
|
print NEWINI "\n";
|
|
}
|
|
|
|
readline(OLDINI);
|
|
} elsif (/weight-t/) {
|
|
print NEWINI "[weight-t]\n";
|
|
foreach my $pm_weight (@pm_weights) {
|
|
print NEWINI $pm_weight;
|
|
print NEWINI "\n";
|
|
readline(OLDINI);
|
|
}
|
|
} elsif (/weight-d/) {
|
|
print NEWINI "[weight-d]\n";
|
|
print NEWINI $d_weight;
|
|
print NEWINI "\n";
|
|
readline(OLDINI);
|
|
foreach my $lr_weight (@lr_weights) {
|
|
print NEWINI $lr_weight;
|
|
print NEWINI "\n";
|
|
readline(OLDINI);
|
|
}
|
|
} elsif (/weight-w/) {
|
|
print NEWINI "[weight-w]\n";
|
|
print NEWINI $wp_weight;
|
|
print NEWINI "\n";
|
|
readline(OLDINI);
|
|
}
|
|
else {
|
|
print NEWINI;
|
|
}
|
|
}
|
|
if ($num_extra_weights > 0) {
|
|
print NEWINI "\n[weight-file]\n$sparse_weights\n";
|
|
}
|
|
close OLDINI;
|
|
close NEWINI;
|
|
print STDERR "new ini file: ".$ini_continue."\n";
|
|
$moses_ini_file = $ini_continue;
|
|
}
|
|
}
|
|
|
|
my $trainer_exe = ¶m_required("train.trainer");
|
|
&check_exists("Training executable", $trainer_exe);
|
|
#my $weights_file = ¶m_required("train.weights-file");
|
|
#&check_exists("weights file ", $weights_file);
|
|
|
|
#optional training parameters
|
|
my $epochs = ¶m("train.epochs");
|
|
my $learner = ¶m("train.learner", "mira");
|
|
my $batch = ¶m("train.batch", 1); # don't print this param twice (when printing training file)
|
|
my $extra_args = ¶m("train.extra-args");
|
|
my $by_node = ¶m("train.by-node");
|
|
my $slots = ¶m("train.slots",10);
|
|
my $jobs = ¶m("train.jobs",10);
|
|
my $mixing_frequency = ¶m("train.mixing-frequency", 1); # don't print this param twice
|
|
my $weight_dump_frequency = ¶m("train.weight-dump-frequency", 1); # don't print this param twice
|
|
my $burn_in = ¶m("train.burn-in");
|
|
my $burn_in_input_file = ¶m("train.burn-in-input-file");
|
|
my $burn_in_reference_files = ¶m("train.burn-in-reference-files");
|
|
my $skipTrain = ¶m("train.skip");
|
|
my $train_decoder_settings = ¶m("train.decoder-settings", "");
|
|
if (!$train_decoder_settings) {
|
|
$train_decoder_settings = $general_decoder_settings;
|
|
}
|
|
|
|
#devtest configuration
|
|
my ($devtest_input_file, $devtest_reference_files,$devtest_ini_file,$bleu_script,$use_moses);
|
|
my $test_exe = ¶m("devtest.moses");
|
|
&check_exists("test executable", $test_exe);
|
|
$bleu_script = ¶m_required("devtest.bleu");
|
|
&check_exists("multi-bleu script", $bleu_script);
|
|
$devtest_input_file = ¶m_required("devtest.input-file");
|
|
&check_exists ("devtest input file", $devtest_input_file);
|
|
$devtest_reference_files = ¶m_required("devtest.reference-file");
|
|
if (&check_exists_noThrow ("devtest ref file", $devtest_reference_files) != 0) {
|
|
for my $ref (glob $devtest_reference_files . "*") {
|
|
&check_exists ("devtest ref file", $ref);
|
|
}
|
|
}
|
|
$devtest_ini_file = ¶m_required("devtest.moses-ini-file");
|
|
&check_exists ("devtest ini file", $devtest_ini_file);
|
|
|
|
|
|
my $weight_file_stem = "$name-weights";
|
|
my $extra_memory_devtest = ¶m("devtest.extra-memory",0);
|
|
my $skip_devtest = ¶m("devtest.skip-devtest",0);
|
|
my $skip_dev = ¶m("devtest.skip-dev",0);
|
|
my $skip_submit_test = ¶m("devtest.skip-submit",0);
|
|
my $devtest_decoder_settings = ¶m("devtest.decoder-settings", "");
|
|
if (!$devtest_decoder_settings) {
|
|
$devtest_decoder_settings = $general_decoder_settings;
|
|
}
|
|
|
|
|
|
# check that number of jobs, dump frequency and number of input sentences are compatible
|
|
# shard size = number of input sentences / number of jobs, ensure shard size >= dump frequency
|
|
if ($jackknife) {
|
|
# TODO..
|
|
}
|
|
else {
|
|
my $result = `wc -l $input_file`;
|
|
my @result = split(/\s/, $result);
|
|
my $inputSize = $result[0];
|
|
my $shardSize = $inputSize / $jobs;
|
|
if ($mixing_frequency) {
|
|
if ($shardSize < $mixing_frequency) {
|
|
$mixing_frequency = int($shardSize);
|
|
if ($mixing_frequency == 0) {
|
|
$mixing_frequency = 1;
|
|
}
|
|
|
|
print STDERR "Warning: mixing frequency must not be larger than shard size, setting mixing frequency to $mixing_frequency\n";
|
|
}
|
|
}
|
|
|
|
if ($weight_dump_frequency != 0) {
|
|
if ($shardSize < $weight_dump_frequency) {
|
|
$weight_dump_frequency = int($shardSize);
|
|
if ($weight_dump_frequency == 0) {
|
|
$weight_dump_frequency = 1;
|
|
}
|
|
|
|
print STDERR "Warning: weight dump frequency must not be larger than shard size, setting weight dump frequency to $weight_dump_frequency\n";
|
|
}
|
|
}
|
|
|
|
if ($mixing_frequency != 0) {
|
|
if ($mixing_frequency > ($shardSize/$batch)) {
|
|
$mixing_frequency = int($shardSize/$batch);
|
|
if ($mixing_frequency == 0) {
|
|
$mixing_frequency = 1;
|
|
}
|
|
|
|
print STDERR "Warning: mixing frequency must not be larger than (shard size/batch size), setting mixing frequency to $mixing_frequency\n";
|
|
}
|
|
}
|
|
|
|
if ($weight_dump_frequency != 0) {
|
|
if ($weight_dump_frequency > ($shardSize/$batch)) {
|
|
$weight_dump_frequency = int($shardSize/$batch);
|
|
if ($weight_dump_frequency == 0) {
|
|
$weight_dump_frequency = 1;
|
|
}
|
|
|
|
print STDERR "Warning: weight dump frequency must not be larger than (shard size/batch size), setting weight dump frequency to $weight_dump_frequency\n";
|
|
}
|
|
}
|
|
}
|
|
|
|
#file names
|
|
my $train_script_file = $working_dir . "/" . $train_script . ".sh";
|
|
my $train_out = $train_script . ".out";
|
|
my $train_err = $train_script . ".err";
|
|
my $train_job_id = 0;
|
|
|
|
my @refs;
|
|
if (ref($reference_files) eq 'ARRAY') {
|
|
@refs = @$reference_files;
|
|
} elsif ($singleRef){
|
|
$refs[0] = $reference_files;
|
|
} else {
|
|
@refs = glob $reference_files . "*"
|
|
}
|
|
my $arr_refs = \@refs;
|
|
|
|
if (!$skipTrain) {
|
|
#write the script
|
|
open TRAIN, ">$train_script_file" or die "Unable to open \"$train_script_file\" for writing";
|
|
|
|
&header(*TRAIN,$job_name,$working_dir,$slots,$jobs,$hours,$vmem,$train_out,$train_err);
|
|
if ($jobs == 1) {
|
|
print TRAIN "$trainer_exe ";
|
|
}
|
|
else {
|
|
if ($by_node) {
|
|
print TRAIN "mpirun -np $jobs --bynode $trainer_exe \\\n";
|
|
}
|
|
else {
|
|
print TRAIN "mpirun -np \$NSLOTS $trainer_exe \\\n";
|
|
}
|
|
}
|
|
|
|
if ($jackknife) {
|
|
foreach my $ini (@moses_ini_files_folds) {
|
|
print TRAIN "--configs-folds $ini ";
|
|
}
|
|
print TRAIN "\\\n";
|
|
foreach my $in (@input_files_folds) {
|
|
print TRAIN "--input-files-folds $in ";
|
|
}
|
|
print TRAIN "\\\n";
|
|
for my $ref (@reference_files_folds) {
|
|
print TRAIN "--reference-files-folds $ref ";
|
|
}
|
|
print TRAIN "\\\n";
|
|
}
|
|
else {
|
|
print TRAIN "-f $moses_ini_file \\\n";
|
|
print TRAIN "-i $input_file \\\n";
|
|
for my $ref (@refs) {
|
|
print TRAIN "-r $ref ";
|
|
}
|
|
print TRAIN "\\\n";
|
|
}
|
|
if ($continue_epoch > 0) {
|
|
print TRAIN "--continue-epoch $continue_epoch \\\n";
|
|
}
|
|
if ($burn_in) {
|
|
print TRAIN "--burn-in 1 \\\n";
|
|
print TRAIN "--burn-in-input-file $burn_in_input_file \\\n";
|
|
my @burnin_refs;
|
|
if (ref($burn_in_reference_files) eq 'ARRAY') {
|
|
@burnin_refs = @$burn_in_reference_files;
|
|
} else {
|
|
@burnin_refs = glob $burn_in_reference_files . "*"; # TODO:
|
|
}
|
|
for my $burnin_ref (@burnin_refs) {
|
|
&check_exists("burn-in ref file", $burnin_ref);
|
|
print TRAIN "--burn-in-reference-files $burnin_ref ";
|
|
}
|
|
print TRAIN "\\\n";
|
|
}
|
|
#if ($weights_file) {
|
|
# print TRAIN "-w $weights_file \\\n";
|
|
#}
|
|
if (defined $start_weight_file) {
|
|
print TRAIN "--start-weights $start_weight_file \\\n";
|
|
}
|
|
print TRAIN "-l $learner \\\n";
|
|
print TRAIN "--weight-dump-stem $weight_file_stem \\\n";
|
|
print TRAIN "--mixing-frequency $mixing_frequency \\\n" if ($extra_args !~ /--mixing-frequency /);
|
|
print TRAIN "--weight-dump-frequency $weight_dump_frequency \\\n" if ($extra_args !~ /--weight-dump-frequency /);
|
|
print TRAIN "--epochs $epochs \\\n" if $epochs;
|
|
print TRAIN "--batch-size $batch \\\n" if ($extra_args !~ /--batch-size / && $extra_args !~ /-b /);
|
|
print TRAIN $extra_args." \\\n";
|
|
print TRAIN "--decoder-settings \"$train_decoder_settings\" \\\n";
|
|
if ($jobs == 1) {
|
|
print TRAIN "echo \"mira finished.\"\n";
|
|
}
|
|
else {
|
|
print TRAIN "echo \"mpirun finished.\"\n";
|
|
}
|
|
close TRAIN;
|
|
|
|
if (! $execute) {
|
|
print STDERR "Written train file: $train_script_file\n";
|
|
exit 0;
|
|
}
|
|
|
|
#submit the training job
|
|
if ($have_sge) {
|
|
$train_job_id = &submit_job_sge($train_script_file);
|
|
|
|
} else {
|
|
$train_job_id = &submit_job_no_sge($train_script_file, $train_out,$train_err);
|
|
}
|
|
|
|
die "Failed to submit training job" unless $train_job_id;
|
|
}
|
|
|
|
#wait for the next weights file to appear, or the training job to end
|
|
my $train_iteration = -1;
|
|
if ($continue_epoch > 0) {
|
|
$train_iteration += ($continue_epoch*$weight_dump_frequency);
|
|
print STDERR "Start from training iteration ".$train_iteration." instead of -1.\n";
|
|
}
|
|
else {
|
|
print STDERR "Start from training iteration ".$train_iteration."\n";
|
|
}
|
|
|
|
while(1) {
|
|
my($epoch, $epoch_slice);
|
|
$train_iteration += 1; # starts at 0
|
|
my $new_weight_file = "$working_dir/$weight_file_stem" . "_";
|
|
if ($weight_dump_frequency == 0) {
|
|
print STDERR "No weights, no testing..\n";
|
|
exit(0);
|
|
}
|
|
|
|
#my $epoch = 1 + int $train_iteration / $weight_dump_frequency;
|
|
$epoch = int $train_iteration / $weight_dump_frequency;
|
|
$epoch_slice = $train_iteration % $weight_dump_frequency;
|
|
if ($weight_dump_frequency == 1) {
|
|
if ($train_iteration < 10) {
|
|
$new_weight_file .= "0".$train_iteration;
|
|
}
|
|
else {
|
|
$new_weight_file .= $train_iteration;
|
|
}
|
|
} else {
|
|
if ($epoch < 10) {
|
|
$new_weight_file .= "0".$epoch."_".$epoch_slice;
|
|
}
|
|
else {
|
|
$new_weight_file .= $epoch."_".$epoch_slice;
|
|
}
|
|
}
|
|
|
|
print STDERR "Current epoch: ".$epoch."\n";
|
|
my $expected_num_files = $epoch*$weight_dump_frequency;
|
|
if ($wait_for_bleu) {
|
|
print STDERR "Expected number of BLEU files: $expected_num_files \n";
|
|
}
|
|
if (-e "$working_dir/stopping") {
|
|
wait_for_bleu($expected_num_files, $working_dir) if ($wait_for_bleu);
|
|
print STDERR "Training finished at " . scalar(localtime()) . " because stopping criterion was reached.\n";
|
|
exit 0;
|
|
}
|
|
elsif (-e "$working_dir/finished") {
|
|
wait_for_bleu($expected_num_files, $working_dir) if ($wait_for_bleu);
|
|
print STDERR "Training finished at " . scalar(localtime()) . " because maximum number of epochs was reached.\n";
|
|
exit 0;
|
|
}
|
|
else {
|
|
print STDERR "Waiting for $new_weight_file\n";
|
|
if (!$skipTrain) {
|
|
while ((! -e $new_weight_file) && &check_running($train_job_id)) {
|
|
sleep 10;
|
|
}
|
|
}
|
|
if (! -e $new_weight_file ) {
|
|
if (-e "$working_dir/stopping") {
|
|
wait_for_bleu($expected_num_files, $working_dir) if ($wait_for_bleu);
|
|
print STDERR "Training finished at " . scalar(localtime()) . " because stopping criterion was reached.\n";
|
|
exit 0;
|
|
}
|
|
elsif (-e "$working_dir/finished") {
|
|
wait_for_bleu($expected_num_files, $working_dir) if ($wait_for_bleu);
|
|
print STDERR "Training finished at " . scalar(localtime()) . " because maximum number of epochs was reached.\n";
|
|
exit 0;
|
|
}
|
|
else {
|
|
# training finished with error
|
|
print STDERR "Error: training was aborted at " . scalar(localtime()) . "\n";
|
|
exit 1;
|
|
}
|
|
}
|
|
}
|
|
|
|
#new weight file written. create test script and submit
|
|
my $suffix = "";
|
|
print STDERR "weight file exists? ".(-e $new_weight_file)."\n";
|
|
if (!$skip_devtest) {
|
|
createTestScriptAndSubmit($epoch, $epoch_slice, $new_weight_file, $suffix, "devtest", $devtest_ini_file, $devtest_input_file, $devtest_reference_files, $skip_submit_test);
|
|
|
|
my $regularized_weight_file = $new_weight_file."_reg";
|
|
if (-e $regularized_weight_file) {
|
|
print STDERR "Submitting test script for regularized weights. \n";
|
|
$epoch_slice .= "_reg";
|
|
createTestScriptAndSubmit($epoch, $epoch_slice, $regularized_weight_file, $suffix, "devtest", $devtest_ini_file, $devtest_input_file, $devtest_reference_files, $skip_submit_test);
|
|
}
|
|
}
|
|
if (!$skip_dev) {
|
|
createTestScriptAndSubmit($epoch, $epoch_slice, $new_weight_file, $suffix, "dev", $moses_ini_file, $input_file, $reference_files, $skip_submit_test);
|
|
}
|
|
}
|
|
|
|
sub wait_for_bleu() {
|
|
my $expected_num_files = $_[0];
|
|
my $working_dir = $_[1];
|
|
print STDERR "Waiting for $expected_num_files bleu files..\n";
|
|
print STDERR "Path: $working_dir/*.bleu \n";
|
|
my @bleu_files = glob("$working_dir/*.bleu");
|
|
while (scalar(@bleu_files) < $expected_num_files) {
|
|
sleep 30;
|
|
@bleu_files = glob("$working_dir/*.bleu");
|
|
print STDERR "currently have ".(scalar(@bleu_files))."\n";
|
|
}
|
|
print STDERR "$expected_num_files BLEU files completed, continue.\n";
|
|
}
|
|
|
|
sub createTestScriptAndSubmit {
|
|
my $epoch = $_[0];
|
|
my $epoch_slice = $_[1];
|
|
my $new_weight_file = $_[2];
|
|
my $suffix = $_[3];
|
|
my $testtype = $_[4];
|
|
my $old_ini_file = $_[5];
|
|
my $input_file = $_[6];
|
|
my $reference_file = $_[7];
|
|
my $skip_submit = $_[8];
|
|
|
|
#file names
|
|
my $output_file;
|
|
my $output_error_file;
|
|
my $bleu_file;
|
|
my $file_id = "";
|
|
if ($weight_dump_frequency == 1) {
|
|
if ($train_iteration < 10) {
|
|
$output_file = $working_dir."/".$name."_0".$train_iteration.$suffix."_$testtype".".out";
|
|
$output_error_file = $working_dir."/".$name."_0".$train_iteration.$suffix."_$testtype".".err";
|
|
$bleu_file = $working_dir."/".$name."_0".$train_iteration.$suffix."_$testtype".".bleu";
|
|
$file_id = "0".$train_iteration.$suffix;
|
|
}
|
|
else {
|
|
$output_file = $working_dir."/".$name."_".$train_iteration.$suffix."_$testtype".".out";
|
|
$output_error_file = $working_dir."/".$name."_".$train_iteration.$suffix."_$testtype".".err";
|
|
$bleu_file = $working_dir."/".$name."_".$train_iteration.$suffix."_$testtype".".bleu";
|
|
$file_id = $train_iteration.$suffix;
|
|
}
|
|
}
|
|
else {
|
|
if ($epoch < 10) {
|
|
$output_file = $working_dir."/".$name."_0".$epoch."_".$epoch_slice.$suffix."_$testtype".".out";
|
|
$output_error_file = $working_dir."/".$name."_0".$epoch."_".$epoch_slice.$suffix."_$testtype".".err";
|
|
$bleu_file = $working_dir."/".$name."_0".$epoch."_".$epoch_slice.$suffix."_$testtype".".bleu";
|
|
$file_id = "0".$epoch."_".$epoch_slice.$suffix;
|
|
}
|
|
else {
|
|
$output_file = $working_dir."/".$name."_".$epoch."_".$epoch_slice.$suffix."_$testtype".".out";
|
|
$output_error_file = $working_dir."/".$name."_".$epoch."_".$epoch_slice.$suffix."_$testtype".".err";
|
|
$bleu_file = $working_dir."/".$name."_".$epoch."_".$epoch_slice.$suffix."_$testtype".".bleu";
|
|
$file_id = $epoch."_".$epoch_slice.$suffix;
|
|
}
|
|
}
|
|
|
|
my $job_name = $name."_".$testtype."_".$file_id;
|
|
|
|
my $test_script = "$name-$testtype";
|
|
my $test_script_file = "$working_dir/$test_script.$file_id.sh";
|
|
my $test_out = "$test_script.$file_id.out";
|
|
my $test_err = "$test_script.$file_id.err";
|
|
|
|
if (! (open TEST, ">$test_script_file" )) {
|
|
die "Unable to create test script $test_script_file\n";
|
|
}
|
|
|
|
my $hours = ¶m("test.hours",12);
|
|
my $extra_args = ¶m("test.extra-args");
|
|
|
|
# Splice the weights into the moses ini file.
|
|
my ($default_weight,$wordpenalty_weight,@phrasemodel_weights,$lm_weight,$lm2_weight,$distortion_weight,@lexicalreordering_weights);
|
|
|
|
if (! (open WEIGHTS, "$new_weight_file")) {
|
|
die "Unable to open weights file $new_weight_file\n";
|
|
}
|
|
|
|
my $readCoreWeights = 0;
|
|
my $readExtraWeights = 0;
|
|
my %extra_weights;
|
|
my $abs_weights = 0;
|
|
my $metaFeature_wt_weight = 0;
|
|
my $metaFeature_pp_weight = 0;
|
|
while(<WEIGHTS>) {
|
|
chomp;
|
|
my ($name,$value) = split;
|
|
next if ($name =~ /^!Unknown/);
|
|
next if ($name =~ /^BleuScore/);
|
|
if ($name eq "DEFAULT_") {
|
|
$default_weight = $value;
|
|
} else {
|
|
if ($name eq "WordPenalty") {
|
|
$wordpenalty_weight = $value;
|
|
$abs_weights += abs($value);
|
|
$readCoreWeights += 1;
|
|
} elsif ($name =~ /^PhraseModel/) {
|
|
push @phrasemodel_weights,$value;
|
|
$abs_weights += abs($value);
|
|
$readCoreWeights += 1;
|
|
} elsif ($name =~ /^LM\:2/) {
|
|
$lm2_weight = $value;
|
|
$abs_weights += abs($value);
|
|
$readCoreWeights += 1;
|
|
}
|
|
elsif ($name =~ /^LM/) {
|
|
$lm_weight = $value;
|
|
$abs_weights += abs($value);
|
|
$readCoreWeights += 1;
|
|
} elsif ($name eq "Distortion") {
|
|
$distortion_weight = $value;
|
|
$abs_weights += abs($value);
|
|
$readCoreWeights += 1;
|
|
} elsif ($name =~ /^LexicalReordering/) {
|
|
push @lexicalreordering_weights,$value;
|
|
$abs_weights += abs($value);
|
|
$readCoreWeights += 1;
|
|
} elsif ($name =~ /^MetaFeature_wt/) {
|
|
$metaFeature_wt_weight = $value;
|
|
$abs_weights += abs($value);
|
|
$readCoreWeights += 1;
|
|
}
|
|
elsif ($name =~ /^MetaFeature_pp/) {
|
|
$metaFeature_pp_weight = $value;
|
|
$abs_weights += abs($value);
|
|
$readCoreWeights += 1;
|
|
}
|
|
else {
|
|
$extra_weights{$name} = $value;
|
|
$readExtraWeights += 1;
|
|
}
|
|
}
|
|
}
|
|
close WEIGHTS;
|
|
|
|
print STDERR "Number of core weights read: ".$readCoreWeights."\n";
|
|
print STDERR "Number of extra weights read: ".$readExtraWeights."\n";
|
|
|
|
# Create new ini file (changing format: expt1-devtest.00_2.ini instead of expt1-devtest.3.ini)
|
|
# my $new_ini_file = $working_dir."/".$test_script.".".$train_iteration.$suffix.".ini";
|
|
my $new_ini_file = "$working_dir/$test_script.$file_id.ini";
|
|
if (! (open NEWINI, ">$new_ini_file" )) {
|
|
die "Unable to create ini file $new_ini_file\n";
|
|
}
|
|
if (! (open OLDINI, "$old_ini_file" )) {
|
|
die "Unable to read ini file $old_ini_file\n";
|
|
}
|
|
|
|
# write normalized weights to ini file
|
|
while(<OLDINI>) {
|
|
if (/weight-l/) {
|
|
print NEWINI "[weight-l]\n";
|
|
print NEWINI ($lm_weight/$abs_weights);
|
|
print NEWINI "\n";
|
|
|
|
if (defined $lm2_weight) {
|
|
readline(OLDINI);
|
|
print NEWINI ($lm2_weight/$abs_weights);
|
|
print NEWINI "\n";
|
|
}
|
|
|
|
readline(OLDINI);
|
|
} elsif (/weight-t/) {
|
|
print NEWINI "[weight-t]\n";
|
|
foreach my $phrasemodel_weight (@phrasemodel_weights) {
|
|
print NEWINI ($phrasemodel_weight/$abs_weights);
|
|
print NEWINI "\n";
|
|
readline(OLDINI);
|
|
}
|
|
} elsif (/weight-d/) {
|
|
print NEWINI "[weight-d]\n";
|
|
print NEWINI ($distortion_weight/$abs_weights);
|
|
print NEWINI "\n";
|
|
readline(OLDINI);
|
|
foreach my $lexicalreordering_weight (@lexicalreordering_weights) {
|
|
print NEWINI ($lexicalreordering_weight/$abs_weights);
|
|
print NEWINI "\n";
|
|
readline(OLDINI);
|
|
}
|
|
} elsif (/weight-wt/) {
|
|
print NEWINI "[weight-wt]\n";
|
|
print NEWINI $metaFeature_wt_weight/$abs_weights;
|
|
print NEWINI "\n";
|
|
readline(OLDINI);
|
|
} elsif (/weight-pp/) {
|
|
print NEWINI "[weight-pp]\n";
|
|
print NEWINI $metaFeature_pp_weight/$abs_weights;
|
|
print NEWINI "\n";
|
|
readline(OLDINI);
|
|
}
|
|
elsif (/weight-w/) {
|
|
print NEWINI "[weight-w]\n";
|
|
print NEWINI ($wordpenalty_weight/$abs_weights);
|
|
print NEWINI "\n";
|
|
readline(OLDINI);
|
|
}
|
|
else {
|
|
print NEWINI;
|
|
}
|
|
}
|
|
close OLDINI;
|
|
|
|
my $writtenExtraWeights = 0;
|
|
|
|
# if there are any non-core weights, write them to a weights file (normalized)
|
|
my $extra_weight_file = undef;
|
|
if (%extra_weights && !$tuneMetaFeature) {
|
|
$extra_weight_file = "$new_weight_file.sparse.scaled";
|
|
if (! (open EXTRAWEIGHT,">$extra_weight_file")) {
|
|
print "Warning: unable to create extra weights file $extra_weight_file";
|
|
next;
|
|
}
|
|
# my $core_weight = 1;
|
|
# if ($have_core) {
|
|
# $default_weight = $extra_weights{"DEFAULT_"};
|
|
# $core_weight = $extra_weights{"core"};
|
|
# }
|
|
foreach my $name (sort keys %extra_weights) {
|
|
next if ($name eq "core");
|
|
next if ($name eq "DEFAULT_");
|
|
my $value = $extra_weights{$name}/$abs_weights;
|
|
|
|
# write only non-zero feature weights to file
|
|
if ($value) {
|
|
# $value /= $core_weight;
|
|
print EXTRAWEIGHT "$name $value\n";
|
|
$writtenExtraWeights += 1;
|
|
}
|
|
}
|
|
}
|
|
|
|
# add specification of sparse weight file to ini
|
|
if (!$tuneMetaFeature) {
|
|
print NEWINI "\n[weight-file] \n";
|
|
print NEWINI "$extra_weight_file \n";
|
|
close NEWINI;
|
|
}
|
|
|
|
print TEST "#!/bin/sh\n";
|
|
print TEST "#\$ -N $job_name\n";
|
|
print TEST "#\$ -wd $working_dir\n";
|
|
print TEST "#\$ -l h_rt=$hours:00:00\n";
|
|
print TEST "#\$ -o $test_out\n";
|
|
print TEST "#\$ -e $test_err\n";
|
|
print TEST "\n";
|
|
if ($have_sge) {
|
|
# some eddie specific stuff
|
|
print TEST ". /etc/profile.d/modules.sh\n";
|
|
print TEST "module load openmpi/ethernet/gcc/latest\n";
|
|
print TEST "export LD_LIBRARY_PATH=/exports/informatics/inf_iccs_smt/shared/boost/lib:\$LD_LIBRARY_PATH\n";
|
|
}
|
|
print TEST "$test_exe $devtest_decoder_settings -i $input_file -f $new_ini_file ";
|
|
# now written to ini file
|
|
# if ($extra_weight_file) {
|
|
# print TEST "-weight-file $extra_weight_file ";
|
|
# }
|
|
print TEST $extra_args;
|
|
print TEST " 1> $output_file 2> $output_error_file\n";
|
|
print TEST "echo \"Decoding of ".$testtype." set finished.\"\n";
|
|
print TEST "$bleu_script $reference_file < $output_file > $bleu_file\n";
|
|
print TEST "echo \"Computed BLEU score of ".$testtype." set.\"\n";
|
|
close TEST;
|
|
|
|
#launch testing
|
|
if(!$skip_submit) {
|
|
if ($have_sge) {
|
|
if ($extra_memory_devtest) {
|
|
print STDERR "Extra memory for test job: $extra_memory_devtest \n";
|
|
&submit_job_sge_extra_memory($test_script_file,$extra_memory_devtest);
|
|
}
|
|
else {
|
|
&submit_job_sge($test_script_file);
|
|
}
|
|
} else {
|
|
&submit_job_no_sge($test_script_file, $test_out,$test_err);
|
|
}
|
|
}
|
|
}
|
|
|
|
sub param {
|
|
my ($key,$default) = @_;
|
|
my $value = $config->param($key);
|
|
$value = $default if !$value;
|
|
# Empty arguments get interpreted as arrays
|
|
$value = 0 if (ref($value) eq 'ARRAY' && scalar(@$value) == 0);
|
|
return $value;
|
|
}
|
|
|
|
sub param_required {
|
|
my ($key) = @_;
|
|
my $value = ¶m($key);
|
|
die "Error: required parameter \"$key\" was missing" if (!defined($value));
|
|
#$value = join $value if (ref($value) eq 'ARRAY');
|
|
return $value;
|
|
}
|
|
|
|
sub header {
|
|
my ($OUT,$name,$working_dir,$slots,$jobs,$hours,$vmem,$out,$err) = @_;
|
|
print $OUT "#!/bin/sh\n";
|
|
if ($have_sge) {
|
|
print $OUT "#\$ -N $name\n";
|
|
print $OUT "#\$ -wd $working_dir\n";
|
|
if ($jobs != 1) {
|
|
print $OUT "#\$ -pe $mpienv $slots\n";
|
|
}
|
|
print $OUT "#\$ -l h_rt=$hours:00:00\n";
|
|
print $OUT "#\$ -l h_vmem=$vmem" . "G" . "\n";
|
|
print $OUT "#\$ -o $out\n";
|
|
print $OUT "#\$ -e $err\n";
|
|
} else {
|
|
print $OUT "\nNSLOTS=$jobs\n";
|
|
}
|
|
print $OUT "\n";
|
|
if ($have_sge) {
|
|
# some eddie specific stuff
|
|
print $OUT ". /etc/profile.d/modules.sh\n";
|
|
print $OUT "module load openmpi/ethernet/gcc/latest\n";
|
|
print $OUT "export LD_LIBRARY_PATH=/exports/informatics/inf_iccs_smt/shared/boost/lib:\$LD_LIBRARY_PATH\n";
|
|
}
|
|
}
|
|
|
|
sub check_exists {
|
|
my ($name,$filename) = @_;
|
|
die "Error: unable to read $name: \"$filename\"" if ! -r $filename;
|
|
}
|
|
|
|
sub check_exists_noThrow {
|
|
my ($name,$filename) = @_;
|
|
return 1 if ! -r $filename;
|
|
return 0;
|
|
}
|
|
|
|
#
|
|
# Used to submit train/test jobs. Return the job id, or 0 on failure
|
|
#
|
|
|
|
sub submit_job_sge {
|
|
my($script_file) = @_;
|
|
my $qsub_result = `qsub -P $queue $script_file`;
|
|
print STDERR "SUBMIT CMD: qsub -P $queue $script_file\n";
|
|
if ($qsub_result !~ /Your job (\d+)/) {
|
|
print STDERR "Failed to qsub job: $qsub_result\n";
|
|
return 0;
|
|
}
|
|
my $job_name = basename($script_file);
|
|
print STDERR "Submitted job: $job_name id: $1 " .
|
|
scalar(localtime()) . "\n";
|
|
return $1;
|
|
}
|
|
|
|
sub submit_job_sge_extra_memory {
|
|
my($script_file,$extra_memory) = @_;
|
|
my $qsub_result = `qsub -pe $extra_memory -P $queue $script_file`;
|
|
print STDERR "SUBMIT CMD: qsub -pe $extra_memory -P $queue $script_file \n";
|
|
if ($qsub_result !~ /Your job (\d+)/) {
|
|
print STDERR "Failed to qsub job: $qsub_result\n";
|
|
return 0;
|
|
}
|
|
my $job_name = basename($script_file);
|
|
print STDERR "Submitted job: $job_name id: $1 " .
|
|
scalar(localtime()) . "\n";
|
|
return $1;
|
|
}
|
|
|
|
#
|
|
# As above, but no sge version. Returns the pid.
|
|
#
|
|
|
|
sub submit_job_no_sge {
|
|
my($script_file,$out,$err) = @_;
|
|
my $pid = undef;
|
|
if ($pid = fork) {
|
|
my $job_name = basename($script_file);
|
|
print STDERR "Launched : $job_name pid: $pid " . scalar(localtime()) . "\n";
|
|
return $pid;
|
|
} elsif (defined $pid) {
|
|
print STDERR "Executing script $script_file, writing to $out and $err.\n";
|
|
`cd $working_dir; sh $script_file 1>$out 2> $err`;
|
|
exit;
|
|
} else {
|
|
# Fork failed
|
|
return 0;
|
|
}
|
|
}
|
|
|
|
sub check_running {
|
|
my ($job_id) = @_;
|
|
if ($have_sge) {
|
|
return `qstat | grep $job_id`;
|
|
} else {
|
|
return `ps -e | grep $job_id | grep -v defunct`;
|
|
}
|
|
}
|
|
|
|
|
|
|