moses-smt/mosesdecoder
1
1
Fork 0
mirror of https://github.com/moses-smt/mosesdecoder.git synced 2024-11-10 10:59:21 +03:00
Code Issues Packages Projects Releases Wiki Activity

update scripts training-expt.perl, extractInformation.perl, mapWeightsToIni.perl

Browse Source 
git-svn-id: http://svn.statmt.org/repository/mira@3797 cc96ff50-19ce-11e0-b349-13d7f0bd23df
This commit is contained in:
ehasler 2011-01-07 18:19:11 +00:00 committed by Ondrej Bojar
parent 93797cb5eb
commit 8309fa8dd7
3 changed files with 264 additions and 202 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

144
mira/extractInformation.perl
View File

@ -1,113 +1,67 @@
#!/usr/bin/env perl
#!/usr/bin/perl
use strict;
my $in = $ARGV[0];
my $out_dir = "extractedInformation";
if (not -d $out_dir) {
mkdir($out_dir);
}
open IN, "<$in" or die "Unable to open \"$in\" for reading";
open OUT1, ">precision1.txt";
open OUT2, ">precision2.txt";
open OUT3, ">precision3.txt";
open OUT4, ">precision4.txt";
open OUT5, ">oracleToReferenceRatio.txt";
open OUT6, ">oracleToReferenceRatioInHistory.txt";
open OUT1, ">$out_dir/newWeights.txt";
open OUT2, ">$out_dir/oracleReferenceRatio.txt";
open OUT3, ">$out_dir/errorAfterMixing.txt";
open OUT4, ">$out_dir/errorAfterDumping.txt";
open OUT7, ">sourceReferenceRatio.txt";
open OUT8, ">hypothesisReferenceRatio.txt";
open OUT9, ">BP.txt";
open OUT5, ">$out_dir/rank0_mixing_before.txt";
open OUT6, ">$out_dir/rank0_mixing_after.txt";
open OUT7, ">$out_dir/rank0_dumping_cumulativeWeights.txt";
open OUT8, ">$out_dir/rank0_dumping_totalWeights.txt";
open OUT9, ">$out_dir/rank0_dumping_averageTotalWeights.txt";
open OUT10, ">trackWeightChanges.txt";
open OUT11, ">sourceOracleRatio.txt";
my $averagePrecision1 = 0;
my $averagePrecision2 = 0;
my $averagePrecision3 = 0;
my $averagePrecision4 = 0;
my $examples = 0;
my $oracleToReferenceAverage = 0;
my $oracleToReferenceHistoryAverage = 0;
my $examples2 = 0;
my $examples3 = 0;
my $sourceReferenceRatio = 0;
my $hypoReferenceRatio = 0;
my $BP = 0;
my $sourceOracleRatio = 0;
while(<IN>){
# if ($_ =~ "precisionHistory 1: (.*) .*\n"){
# print OUT1 "$1\n";
# $averagePrecision1 += $1;
# }
# elsif ($_ =~ "precisionHistory 2: (.*) .*\n"){
# print OUT2 "$1\n";
# $averagePrecision2 += $1;
# }
# elsif ($_ =~ "precisionHistory 3: (.*) .*\n"){
# print OUT3 "$1\n";
# $averagePrecision3 += $1;
# }
# elsif ($_ =~ "precisionHistory 4: (.*) .*\n"){
# print OUT4 "$1\n";
# $averagePrecision4 += $1;
# }
# elsif ($_ =~ "ratio length: (.*)\n"){
# # oracle to reference ratio
# print OUT5 "$1\n";
# $oracleToReferenceAverage += $1;
# }
# elsif ($_ =~ "ratio history: (.*)\n"){
# # oracle to reference ratio as in the history
# print OUT6 "$1\n";
# $oracleToReferenceHistoryAverage += $1;
# }
if ($_ =~ "source/reference ratio: (.*)\n"){
print OUT7 "$1\n";
#$examples += 1;
$sourceReferenceRatio += $1;
}
elsif ($_ =~ "hypo length/reference length: (.*)\n") {
print OUT8 "$1\n";
$examples2 += 1;
$hypoReferenceRatio += $1;
}
elsif ($_ =~ "BP: (.*)\n") {
print OUT9 "$1\n";
$examples3 += 1;
$BP += $1;
}
#elsif ($_ =~ "New weights: <<\{((-?\d*\.\d*), ){1,}\}>>\n") {
elsif ($_ =~ "New weights: <<\{((?:.*?, ){1,})0, \}>>\n") {
if ($_ =~ m/New weights: <<\{((?:\-?\d*?\.?\d*?, ){1,})0, \}>>/) {
my @weights = split(/,/, $1);
print OUT10 "@weights\n";
print OUT1 "@weights\n";
}
elsif ($_ =~ "source/oracle ratio: (.*)\n") {
print OUT11 "$1\n";
$sourceOracleRatio += $1;
$examples += 1;
elsif ($_ =~ m/oracle length: (\d*) reference length: (\d*)/) {
my $ratio = $1/$2;
print OUT2 "$ratio\n";
}
elsif ($_ =~ m/summed error after mixing weights: (\d*?\.?\d*)/) {
print OUT3 "$1\n";
}
elsif ($_ =~ m/summed error after dumping weights: (\d*?\.\d*)/) {
print OUT4 "$1\n";
}
elsif ($_ =~ m/summed error after dumping weights: (\d*?\.?\d*)/) {
print OUT4 "$1\n";
}
elsif ($_ =~ m/Rank 0, before mixing: <<\{((?:.*?, ){1,})0, \}>>/) {
my @weights = split(/,/, $1);
print OUT5 "@weights\n";
}
elsif ($_ =~ m/Rank 0, after mixing: <<\{((?:.*?, ){1,})0, \}>>/) {
my @weights = split(/,/, $1);
print OUT6 "@weights\n";
}
elsif ($_ =~ m/Rank 0, cumulative weights: <<\{((?:.*?, ){1,})0, \}>>/) {
my @weights = split(/,/, $1);
print OUT7 "@weights\n";
}
elsif ($_ =~ m/Rank 0, total weights: <<\{((?:.*?, ){1,})0, \}>>/) {
my @weights = split(/,/, $1);
print OUT8 "@weights\n";
}
elsif ($_ =~ m/Rank 0, average total weights: <<\{((?:.*?, ){1,})0, \}>>/) {
my @weights = split(/,/, $1);
print OUT9 "@weights\n";
}
}
#print OUT1 "average: ", $averagePrecision1/$examples ,"\n";
#print OUT2 "average: ", $averagePrecision2/$examples ,"\n";
#print OUT3 "average: ", $averagePrecision3/$examples ,"\n";
#print OUT4 "average: ", $averagePrecision4/$examples ,"\n";
#print OUT5 "average: ", $oracleToReferenceAverage/$examples, "\n";
#print OUT6 "average: ", $oracleToReferenceHistoryAverage/$examples, "\n";
#print OUT7 "average: ", $sourceReferenceRatio/$examples, "\n";
#print OUT8 "average: ", $hypoReferenceRatio/$examples2, "\n";
#print OUT9 "average: ", $BP/$examples3, "\n";
if ($examples > 0) {
print OUT11 "average: ", $sourceOracleRatio/$examples, "\n";
}

253
mira/mapWeightsToIni.perl
View File

@ -3,114 +3,185 @@
use strict;
#eddie specific
use lib "/exports/informatics/inf_iccs_smt/perl/lib/perl5/site_perl";
use File::Copy;
my $weight_file = $ARGV[0];
my $old_ini_file = $ARGV[1];
my $new_ini_file = $ARGV[2];
my ($weight_file, $old_ini_file, $new_ini_file, $input_dir);
my $old_ini_file_copy = "temporary_ini_file";
my %extra_weights;
my $total;
$weight_file = $ARGV[0];
if (-d $weight_file) {
$input_dir = $ARGV[0];
$old_ini_file = $ARGV[1];
print "input dir: $input_dir, old ini file: $old_ini_file\n";
}
else {
$old_ini_file = $ARGV[1];
$new_ini_file = $ARGV[2];
print "weight file: $weight_file, old ini file: $old_ini_file, new ini file: $new_ini_file\n";
}
while (1) {
print "continue? y/n\n";
my $continue = <STDIN>;
if ($continue eq "y\n") {
last;
}
if ($continue eq "n\n") {
exit;
}
}
# Normalise the weights and splice them into the moses ini file.
my ($default_weight,$wordpenalty_weight,$unknownwordpenalty_weight,@phrasemodel_weights,$lm_weight,$distortion_weight,@lexicalreordering_weights);
if (! (open WEIGHTS, "$weight_file")) {
die "Warning: unable to open weights file $weight_file\n";
}
if ($input_dir) {
$old_ini_file_copy = $input_dir."/".$old_ini_file_copy;
my $weights_file_stem = "expt-weights";
opendir(WEIGHTS, "$input_dir") || die("Cannot open directory");
my %extra_weights;
while(<WEIGHTS>) {
chomp;
my ($name,$value) = split;
if ($name eq "DEFAULT_") {
$default_weight = $value;
} else {
if ($name eq "WordPenalty") {
$wordpenalty_weight = $value;
} elsif ($name eq "!UnknownWordPenalty") {
$unknownwordpenalty_weight = $value;
} elsif ($name =~ /^PhraseModel/) {
push @phrasemodel_weights,$value;
} elsif ($name =~ /^LM/) {
$lm_weight = $value;
} elsif ($name eq "Distortion") {
$distortion_weight = $value;
} elsif ($name =~ /^LexicalReordering/) {
push @lexicalreordering_weights,$value;
} elsif ($name ne "BleuScore") {
$extra_weights{$name} = $value;
my @files= readdir(WEIGHTS);
@files = sort(@files);
my $processedFiles = 0;
foreach my $file (@files) {
if ($file =~ $weights_file_stem) {
$weight_file = $input_dir."/".$file;
# read in the weights
readWeights();
# normalise them if necessary
normaliseWeights();
$new_ini_file = $input_dir."/expt-test.".$processedFiles.".ini";
$processedFiles += 1;
# write new ini file
readOldWriteNewIniFile();
}
}
}
close WEIGHTS;
# Normalising factor
my $total = abs($wordpenalty_weight+$default_weight) +
abs($unknownwordpenalty_weight+$default_weight) +
abs($lm_weight+$default_weight) +
abs($distortion_weight+$default_weight);
# if weights are normalized already, do nothing
if ($total == 0) {
$total = 1.0;
$default_weight = 0.0;
if (-e $old_ini_file_copy ) {
unlink($old_ini_file_copy);
}
}
else {
readWeights();
normaliseWeights();
readOldWriteNewIniFile();
}
foreach my $phrasemodel_weight (@phrasemodel_weights) {
$total += abs($phrasemodel_weight + $default_weight);
}
foreach my $lexicalreordering_weight (@lexicalreordering_weights) {
$total += abs($lexicalreordering_weight + $default_weight);
}
# read old ini file
if (! (open OLDINI, "$old_ini_file" )) {
die "Warning: unable to read old ini file $old_ini_file\n";
}
# Create new ini file
if (! (open NEWINI, ">$new_ini_file" )) {
die "Warning: unable to create new ini file: specify name!\n";
}
while(<OLDINI>) {
if (/weight-l/) {
print NEWINI "[weight-l]\n";
print NEWINI ($lm_weight+$default_weight) / $total;
print NEWINI "\n";
readline(OLDINI);
} elsif (/weight-t/) {
print NEWINI "[weight-t]\n";
foreach my $phrasemodel_weight (@phrasemodel_weights) {
print NEWINI ($phrasemodel_weight+$default_weight) / $total;
print NEWINI "\n";
readline(OLDINI);
sub readWeights {
if (! (open WEIGHTS, "$weight_file")) {
die "Warning: unable to open weights file $weight_file\n";
}
while(<WEIGHTS>) {
chomp;
my ($name,$value) = split;
if ($name eq "DEFAULT_") {
$default_weight = $value;
} else {
if ($name eq "WordPenalty") {
$wordpenalty_weight = $value;
} elsif ($name eq "!UnknownWordPenalty") {
$unknownwordpenalty_weight = $value;
} elsif ($name =~ /^PhraseModel/) {
push @phrasemodel_weights,$value;
} elsif ($name =~ /^LM/) {
$lm_weight = $value;
} elsif ($name eq "Distortion") {
$distortion_weight = $value;
} elsif ($name =~ /^LexicalReordering/) {
push @lexicalreordering_weights,$value;
} elsif ($name ne "BleuScore") {
$extra_weights{$name} = $value;
}
}
} elsif (/weight-d/) {
print NEWINI "[weight-d]\n";
print NEWINI ($distortion_weight+$default_weight) / $total;
print NEWINI "\n";
readline(OLDINI);
foreach my $lexicalreordering_weight (@lexicalreordering_weights) {
print NEWINI ($lexicalreordering_weight+$default_weight) / $total;
}
close WEIGHTS;
}
sub normaliseWeights {
# Normalising factor
$total = abs($wordpenalty_weight+$default_weight) +
abs($unknownwordpenalty_weight+$default_weight) +
abs($lm_weight+$default_weight) +
abs($distortion_weight+$default_weight);
# if weights are normalized already, do nothing
if ($total == 0) {
$total = 1.0;
$default_weight = 0.0;
}
foreach my $phrasemodel_weight (@phrasemodel_weights) {
$total += abs($phrasemodel_weight + $default_weight);
}
foreach my $lexicalreordering_weight (@lexicalreordering_weights) {
$total += abs($lexicalreordering_weight + $default_weight);
}
}
sub readOldWriteNewIniFile {
copy($old_ini_file, $old_ini_file_copy) or die "File $old_ini_file cannot be copied.";
# read old ini file
if (! (open OLDINI, "$old_ini_file_copy" )) {
die "Warning: unable to read old ini file $old_ini_file_copy\n";
}
# Create new ini file
if (! (open NEWINI, ">$new_ini_file" )) {
die "Warning: unable to create new ini file: specify name!\n";
}
while(<OLDINI>) {
if (/weight-l/) {
print NEWINI "[weight-l]\n";
print NEWINI ($lm_weight+$default_weight) / $total;
print NEWINI "\n";
readline(OLDINI);
} elsif (/weight-t/) {
print NEWINI "[weight-t]\n";
foreach my $phrasemodel_weight (@phrasemodel_weights) {
print NEWINI ($phrasemodel_weight+$default_weight) / $total;
print NEWINI "\n";
readline(OLDINI);
}
} elsif (/weight-d/) {
print NEWINI "[weight-d]\n";
print NEWINI ($distortion_weight+$default_weight) / $total;
print NEWINI "\n";
readline(OLDINI);
foreach my $lexicalreordering_weight (@lexicalreordering_weights) {
print NEWINI ($lexicalreordering_weight+$default_weight) / $total;
print NEWINI "\n";
readline(OLDINI);
}
} elsif (/weight-w/) {
print NEWINI "[weight-w]\n";
print NEWINI ($wordpenalty_weight+$default_weight) / $total;
print NEWINI "\n";
readline(OLDINI);
} elsif (/weight-u/) {
print NEWINI "[weight-u]\n";
print NEWINI ($unknownwordpenalty_weight+$default_weight) / $total;
print NEWINI "\n";
readline(OLDINI);
} else {
print NEWINI;
}
} elsif (/weight-w/) {
print NEWINI "[weight-w]\n";
print NEWINI ($wordpenalty_weight+$default_weight) / $total;
print NEWINI "\n";
readline(OLDINI);
} elsif (/weight-u/) {
print NEWINI "[weight-u]\n";
print NEWINI ($unknownwordpenalty_weight+$default_weight) / $total;
print NEWINI "\n";
readline(OLDINI);
} else {
print NEWINI;
}
close NEWINI;
close OLDINI;
@phrasemodel_weights = ();
@lexicalreordering_weights = ();
}
close NEWINI;
close OLDINI;

69
mira/training-expt.perl
View File

@ -81,6 +81,7 @@ my $trainer_exe = &param_required("train.trainer");
#optional training parameters
my $epochs = &param("train.epochs", 2);
my $learner = &param("train.learner", "mira");
my $batch = &param("train.batch", 1);
my $extra_args = &param("train.extra-args");
my $continue_from_epoch = &param("train.continue-from-epoch", 0);
@ -103,6 +104,11 @@ $test_ini_file = &param_required("test.moses-ini-file");
my $weight_file_stem = "$name-weights";
my $weight_frequency = &param("test.frequency",1);
# adjust test frequency when using batches > 1
if ($batch > 1) {
$weight_frequency = 1;
}
# check that number of jobs, dump frequency and number of input sentences are compatible
# shard size = number of input sentences / number of jobs, ensure shard size >= dump frequency
my $result = `wc -l $input_file`;
@ -144,6 +150,7 @@ print TRAIN "-l $learner \\\n";
print TRAIN "--weight-dump-stem $weight_file_stem \\\n";
print TRAIN "--weight-dump-frequency $weight_frequency \\\n";
print TRAIN "--epochs $epochs \\\n";
print TRAIN "-b $batch \\\n";
print TRAIN $extra_args;
print TRAIN "\n";
@ -182,17 +189,30 @@ while(1) {
} else {
#my $epoch = 1 + int $train_iteration / $weight_frequency;
$epoch = int $train_iteration / $weight_frequency;
$epoch_slice = $train_iteration % $weight_frequency;
$new_weight_file .= $epoch . "_" . $epoch_slice;
$epoch_slice = $train_iteration % $weight_frequency;
if ($epoch < 10) {
$new_weight_file .= "0" . $epoch . "_" . $epoch_slice;
}
else {
$new_weight_file .= $epoch . "_" . $epoch_slice;
}
}
print "Waiting for $new_weight_file\n";
while ((! -e $new_weight_file) && &check_running($train_job_id)) {
sleep 10;
}
if (! -e $new_weight_file ) {
print "Training finished at " . scalar(localtime()) . "\n";
if (-e "stopping") {
print "Training finished at " . scalar(localtime()) . " because stopping criterion was reached.\n";
exit 0;
}
else {
print "Waiting for $new_weight_file\n";
while ((! -e $new_weight_file) && &check_running($train_job_id)) {
sleep 10;
}
if (! -e $new_weight_file ) {
print "Training finished at " . scalar(localtime()) . "\n";
exit 0;
}
}
#new weight file written. create test script and submit
#file names
my $job_name = $name . "_$train_iteration";
@ -205,14 +225,28 @@ while(1) {
my $output_error_file;
my $bleu_file;
if ($weight_frequency == 1) {
$output_file = $working_dir."/".$name."_".$train_iteration.".out";
$output_error_file = $working_dir."/".$name."_".$train_iteration.".err";
$bleu_file = $working_dir."/".$name."_".$train_iteration.".bleu";
if ($train_iteration < 10) {
$output_file = $working_dir."/".$name."_0".$train_iteration.".out";
$output_error_file = $working_dir."/".$name."_0".$train_iteration.".err";
$bleu_file = $working_dir."/".$name."_0".$train_iteration.".bleu";
}
else {
$output_file = $working_dir."/".$name."_0".$train_iteration.".out";
$output_error_file = $working_dir."/".$name."_0".$train_iteration.".err";
$bleu_file = $working_dir."/".$name."_0".$train_iteration.".bleu";
}
}
else {
$output_file = $working_dir."/".$name."_".$epoch."_".$epoch_slice.".out";
$output_error_file = $working_dir."/".$name."_".$epoch."_".$epoch_slice.".err";
$bleu_file = $working_dir."/".$name."_".$epoch."_".$epoch_slice.".bleu";
if ($epoch < 10) {
$output_file = $working_dir."/".$name."_0".$epoch."_".$epoch_slice.".out";
$output_error_file = $working_dir."/".$name."_0".$epoch."_".$epoch_slice.".err";
$bleu_file = $working_dir."/".$name."_0".$epoch."_".$epoch_slice.".bleu";
}
else {
$output_file = $working_dir."/".$name."_".$epoch."_".$epoch_slice.".out";
$output_error_file = $working_dir."/".$name."_".$epoch."_".$epoch_slice.".err";
$bleu_file = $working_dir."/".$name."_".$epoch."_".$epoch_slice.".bleu";
}
}
if (! (open TEST, ">$test_script_file" )) {
@ -246,7 +280,7 @@ while(1) {
my %extra_weights;
while(<WEIGHTS>) {
chomp;
my ($name,$value) = split;
my ($name,$value) = split;
#next if ($name =~ /^!Unknown/);
if ($name eq "DEFAULT_") {
$default_weight = $value;
@ -269,6 +303,9 @@ while(1) {
}
}
close WEIGHTS;
die "LM weight not defined" unless defined $lm_weight;
# If there was a core weight file, then we have to load the weights
# from the new weight file
if ($core_weight_file ne $new_weight_file) {
@ -300,7 +337,7 @@ while(1) {
foreach my $lexicalreordering_weight (@lexicalreordering_weights) {
$total += abs($lexicalreordering_weight + $default_weight);
}
# Create new ini file
my $new_test_ini_file = $working_dir . "/" . $test_script . ".$train_iteration.ini";
if (! (open NEWINI, ">$new_test_ini_file" )) {