moses-smt/mosesdecoder
1
1
Fork 0
mirror of https://github.com/moses-smt/mosesdecoder.git synced 2024-12-25 12:52:29 +03:00
Code Issues Packages Projects Releases Wiki Activity

mert-sge-nosync /Raymond Ng

Browse Source
This commit is contained in:
Hieu Hoang 2014-04-30 16:51:26 +01:00
parent ae4ab9aae0
commit 55064961f7
13 changed files with 7200 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

29
contrib/mert-sge-nosync/README Normal file
View File

@ -0,0 +1,29 @@
MERT-sge-nosync
Raymond Ng, University of Sheffield.
Apr, 2014.
The parallel MERT tuning scripts in moses cannot run in the SGE "no-sync" mode (where job submission is done in one go, after which user can go offline while leaving the SGE to manage the whole process). Scripts provided in this site are for parallel MERT in SGE no-sync mode. You will need to have SSH support in perl (Step 2), and set up public ssh-keys between the running hosts and the submission hosts (Step 3).
1. Untar the scipts in ${MOSES}/scripts/
2. Download and install OpenSSH packages for perl:
Net-OpenSSH-Compat (http://search.cpan.org/CPAN/authors/id/S/SA/SALVA/Net-OpenSSH-Compat-0.06.tar.gz)
$ cd Net-OpenSSH-Compat-0.06
$ perl Makefile.PL
(You may have dependency issue and need to install IO-Tty and Net-OpenSSH in advance)
IO-Tty-1.10 (http://search.cpan.org/CPAN/authors/id/T/TO/TODDR/IO-Tty-1.10.tar.gz)
Net-OpenSSH-0.60 (http://search.cpan.org/CPAN/authors/id/S/SA/SALVA/Net-OpenSSH-0.60.tar.gz)
3. Set up public ssh-keys for accessing the submithost (machine from which qsub is executed) from the running hosts (machines which actually runs the scripts)
http://www.linuxproblem.org/art_9.html
4. Run parallel MERT by
nohup nice ${MOSES}/scripts/training/mert-moses-sge-nosync.pl ${TM_DEVTEXT_SOURCE} \
--threads 20 --jobs 20 \
--queue-flags='-q normal.q -P project' \
--submithost='squeal' \
${TM_DEVTEXT_TARGET} ${MOSES}/dist/bin/moses ${WORKINGDIR}/train/model/moses.ini \
--mertdir ${MOSES}/dist/bin/ >& ${WORKINGDIR}/mert.out
(tested on moses version Built Apr 2012 version)

1388
contrib/mert-sge-nosync/generic/moses-parallel-sge-nosync.pl Executable file
View File

File diff suppressed because it is too large Load Diff

312
contrib/mert-sge-nosync/generic/qsub-wrapper-exit-sge-nosync.pl Executable file
View File

@ -0,0 +1,312 @@
#! /usr/bin/perl
# $Id$
use strict;
use Net::OpenSSH::Compat::Perl;
#######################
#Default parameters
#parameters for submiiting processes through SGE
#NOTE: group name is ws06ossmt (with 2 's') and not ws06osmt (with 1 's')
my $queueparameters="";
# look for the correct pwdcmd
my $pwdcmd = getPwdCmd();
my $workingdir = `$pwdcmd`; chomp $workingdir;
# my $tmpdir="$workingdir/tmp$$";
# my $jobscript="$workingdir/job$$.sh";
# my $qsubout="$workingdir/out.job$$";
# my $qsuberr="$workingdir/err.job$$";
$SIG{INT} = \&kill_all_and_quit; # catch exception for CTRL-C
my $submithost="";
my $help="";
my $dbg="";
my $version="";
my $qsubname="WR$$";
my $cmd="";
my $cmdout=undef;
my $cmderr=undef;
my $jid=0;
my $jidfile=undef;
my $pid=0;
my $pidfile=undef;
my $prevjid=undef;
my $parameters="";
my $old_sge = 0; # assume grid engine < 6.0
my $prevjidarraysize = 0;
my $force_delete = 0;
my @prevjidarray = ();
sub init(){
use Getopt::Long qw(:config pass_through);
GetOptions('version'=>\$version,
'help'=>\$help,
'debug'=>\$dbg,
'submithost=s'=> \$submithost,
'qsub-prefix=s'=> \$qsubname,
'stdout=s'=> \$cmdout,
'stderr=s'=> \$cmderr,
'jidfile=s'=> \$jidfile,
'pidfile=s'=> \$pidfile, # process id for previous job
'prevjid=s'=> \$prevjid,
'queue-parameter=s'=> \$queueparameters,
'force-delete=i' => \$force_delete,
'old-sge' => \$old_sge,
) or exit(1);
$parameters="@ARGV";
# read $pid from file
chomp($pid=`tail -n 1 $pidfile`);
# print "PID=+$pidfile+\n";
if (defined $jidfile) {
chomp($jid=`tail -n 1 $jidfile`);
}
# print STDERR "INPUT prevjid =+$prevjid+\n";
$prevjid =~ s/^\s+|\s+$//g;
# print STDERR "TRIMMED prevjid =+$prevjid+\n";
@prevjidarray = split(/\s+/,$prevjid);
$prevjidarraysize = scalar(@prevjidarray);
# print STDERR "arraysize: $prevjidarraysize\n";
version() if $version;
usage() if $help;
print_parameters() if $dbg;
}
#######################
##print version
sub version(){
# print STDERR "version 1.0 (29-07-2006)\n";
print STDERR "version 1.1 (31-07-2006)\n";
exit(1);
}
#usage
sub usage(){
print STDERR "qsub-wrapper.pl [options]\n";
print STDERR "Options:\n";
print STDERR "-stdout <file> file to find stdout from target cmd (optional)\n";
print STDERR "-stderr <file> file to find stderr from target cmd (optional)\n";
print STDERR "-jidfile <file> file to find the submit jobid (for submit option)\n";
print STDERR "-pidfile <file> file to find the process id to the target job for deletion\n";
print STDERR "-prevjid <id> wait for the previous job with jobid=id to finish before starting (optional)\n";
print STDERR "-force-delete 1 force-delete without checking\n";
print STDERR "-qsub-prefix <string> name for sumbitted jobs (optional)\n";
print STDERR "-queue-parameters <string> parameter for the queue (optional)\n";
print STDERR "-old-sge ... assume Sun Grid Engine < 6.0\n";
print STDERR "-debug debug\n";
print STDERR "-version print version of the script\n";
print STDERR "-help this help\n";
exit(1);
}
#printparameters
sub print_parameters(){
# print STDERR "command: $cmd\n";
if (defined($cmdout)){ print STDERR "file for stdout: $cmdout\n"; }
else { print STDERR "file for stdout is not defined, stdout is discarded\n"; }
if (defined($cmderr)){ print STDERR "file for stdout: $cmderr\n"; }
else { print STDERR "file for stderr is not defined, stderr is discarded\n"; }
if (defined($jidfile)){ print STDERR "file for submit job id: $jidfile\n"; }
else { print STDERR "file for submit job id is not defined, jidfile is discarded\n"; }
print STDERR "Qsub name: $qsubname\n";
print STDERR "Queue parameters: $queueparameters\n";
print STDERR "parameters directly passed to cmd: $parameters\n";
exit(1);
}
#######################
#Script starts here
init();
my $tmpdir="$workingdir/tmp$pid";
my $jobscript="$workingdir/job$pid.sh";
my $qsubout="$workingdir/out.job$pid";
my $qsuberr="$workingdir/err.job$pid";
### usage() if $cmd eq "";
###
### safesystem("mkdir -p $tmpdir") or die;
###
### preparing_script();
###
#### my $maysync = $old_sge ? "" : "-sync y";
#### never run in syn mode
###my $maysync = "";
###
###my $qsubcmd = "";
#### create the qsubcmd to submit to the queue with the parameter "-b yes"
####my $qsubcmd="qsub $queueparameters $maysync -V -o $qsubout -e $qsuberr -N $qsubname -b yes $jobscript > $jobscript.log 2>&1";
###
#### add -b yes if not yet defined, otherwise leave empty
###$queueparameters .= " -b yes " if (index($queueparameters," -b ")==-1);
###
###
###if (defined $prevjid && $prevjid!=-1 && $prevjidarraysize == 1) {
### $qsubcmd="qsub $queueparameters $maysync -V -hold_jid $prevjid -o $qsubout -e $qsuberr -N $qsubname $jobscript > $jobscript.log 2>&1";
###} elsif (defined $prevjid && $prevjidarraysize > 1) {
### my $hj = "-hold_jid " . join(" -hold_jid ", @prevjidarray);
### # print STDERR "hj is $hj\n";
### $qsubcmd="qsub $queueparameters $maysync -V $hj -o $qsubout -e $qsuberr -N $qsubname $jobscript > $jobscript.log 2>&1";
###} else {
### $qsubcmd="qsub $queueparameters $maysync -V -o $qsubout -e $qsuberr -N $qsubname $jobscript > $jobscript.log 2>&1";
###}
###
###print "submitting $qsubcmd\n";
###
####run the qsubcmd
###
###safesystem($qsubcmd) or die;
###
####getting id of submitted job#############
###my $res;
###open (IN,"$jobscript.log") or die "Can't read main job id: $jobscript.log";
###chomp($res=<IN>);
###my @arrayStr = split(/\s+/,$res);
###my $id=$arrayStr[2];
###die "Failed to get job id from $jobscript.log, got: $res"
### if $id !~ /^[0-9]+$/;
###close(IN);
############################################
###print STDERR " res:$res\n";
###print STDERR " id:$id\n";
###
###open (JIDOUT,">$jidfile") or die "Can't open jid file to write";
###print JIDOUT "$id\n";
###close(JIDOUT);
###
###open (JOBNUMOUT,">$jidfile.job") or die "Can't open id.job file to write";
###print JOBNUMOUT "$$\n";
###close(JOBNUMOUT);
###
###if ($old_sge) {
### # need to workaround -sync, add another job that will wait for the main one
### # prepare a fake waiting script
### my $syncscript = "$jobscript.sync_workaround_script.sh";
### safesystem("echo 'date' > $syncscript") or die;
###
### my $checkpointfile = "$jobscript.sync_workaround_checkpoint";
###
### # ensure checkpoint does not exist
### safesystem("\\rm -f $checkpointfile") or die;
###
### # start the 'hold' job, i.e. the job that will wait
#### $cmd="qsub -cwd $queueparameters -hold_jid $id -o $checkpointfile f -e /dev/null -N $qsubname.W $syncscript >& $qsubname.W.log";
### $cmd="qsub -cwd $queueparameters -hold_jid $id -o $checkpointfile -e /dev/null -N $qsubname.W $syncscript >& $qsubname.W.log";
### safesystem($cmd) or die;
###
### # and wait for checkpoint file to appear
### my $nr=0;
### while (!-e $checkpointfile) {
### sleep(10);
### $nr++;
### print STDERR "w" if $nr % 3 == 0;
### }
### safesystem("\\rm -f $checkpointfile $syncscript") or die();
### print STDERR "End of waiting workaround.\n";
###}
my $failure=0;
if (!$force_delete) {
$failure=&check_exit_status();
print STDERR "check_exit_status returned $failure\n";
}
&kill_all_and_quit() if $failure;
&remove_temporary_files() if !$dbg;
sub check_exit_status(){
my $failure=0;
print STDERR "check_exit_status of submitted job $jid from file $qsubout\n";
open(IN,"$qsubout") or die "Can't read $qsubout";
while (<IN>){
$failure=1 if (/failed with exit status/);
}
close(IN);
return $failure;
}
sub kill_all_and_quit(){
my $my_username = undef;
# chomp($my_username = `whoami`);
#
# my $ssh = Net::OpenSSH::Compat::Perl->new($submithost, debug=>0);
#
# $ssh->login("$my_username",`cat /home/$my_username/accpw`);
my $ssh = Net::OpenSSH::Compat::Perl->new($submithost, debug=>0);
$ssh->login();
print STDERR "kill_all_and_quit\n";
print STDERR "qdel $jid\n";
# safesystem("qdel $jid");
$ssh->cmd("qdel $jid");
print STDERR "The submitted jobs died not correctly\n";
print STDERR "Send qdel signal to the submitted jobs\n";
exit(1);
}
sub remove_temporary_files(){
#removing temporary files
unlink("${jobscript}");
unlink("${jobscript}.log");
unlink("$qsubout");
unlink("$qsuberr");
rmdir("$tmpdir");
}
sub safesystem {
print STDERR "Executing: @_\n";
system(@_);
if ($? == -1) {
print STDERR "Failed to execute: @_\n $!\n";
exit(1);
}
elsif ($? & 127) {
printf STDERR "Execution of: @_\n died with signal %d, %s coredump\n",
($? & 127), ($? & 128) ? 'with' : 'without';
exit(1);
}
else {
my $exitcode = $? >> 8;
print STDERR "Exit code: $exitcode\n" if $exitcode;
return ! $exitcode;
}
}
# look for the correct pwdcmd (pwd by default, pawd if it exists)
# I assume that pwd always exists
sub getPwdCmd(){
my $pwdcmd="pwd";
my $a;
chomp($a=`which pawd | head -1 | awk '{print $1}'`);
if ($a && -e $a){ $pwdcmd=$a; }
return $pwdcmd;
}

320
contrib/mert-sge-nosync/generic/qsub-wrapper-sge-nosync.pl Executable file
View File

@ -0,0 +1,320 @@
#! /usr/bin/perl
# $Id$
use strict;
#######################
#Default parameters
#parameters for submiiting processes through SGE
#NOTE: group name is ws06ossmt (with 2 's') and not ws06osmt (with 1 's')
my $queueparameters="";
# look for the correct pwdcmd
my $pwdcmd = getPwdCmd();
my $workingdir = `$pwdcmd`; chomp $workingdir;
my $uniqtime = `date +"%s%N"`; chomp $uniqtime;
my $uid = "$$".".".$uniqtime;
my $tmpdir="$workingdir/tmp$uid";
my $jobscript="$workingdir/job$uid.sh";
my $qsubout="$workingdir/out.job$uid";
my $qsuberr="$workingdir/err.job$uid";
$SIG{INT} = \&kill_all_and_quit; # catch exception for CTRL-C
my $help="";
my $dbg="";
my $version="";
my $qsubname="WR$uid";
my $cmd="";
my $cmdout=undef;
my $cmderr=undef;
my $jidfile=undef;
my $pidfile=undef;
my $prevjid=undef;
my $parameters="";
my $old_sge = 0; # assume grid engine < 6.0
my $prevjidarraysize = 0;
my @prevjidarray = ();
sub init(){
use Getopt::Long qw(:config pass_through);
GetOptions('version'=>\$version,
'help'=>\$help,
'debug'=>\$dbg,
'qsub-prefix=s'=> \$qsubname,
'command=s'=> \$cmd,
'stdout=s'=> \$cmdout,
'stderr=s'=> \$cmderr,
'jidfile=s'=> \$jidfile,
'prevjid=s'=> \$prevjid,
'queue-parameters=s'=> \$queueparameters,
'old-sge' => \$old_sge,
) or exit(1);
$parameters="@ARGV";
# print STDERR "INPUT prevjid =+$prevjid+\n";
$prevjid =~ s/^\s+|\s+$//g;
# print STDERR "TRIMMED prevjid =+$prevjid+\n";
@prevjidarray = split(/\s+/,$prevjid);
$prevjidarraysize = scalar(@prevjidarray);
# print STDERR "arraysize: $prevjidarraysize\n";
version() if $version;
usage() if $help;
print_parameters() if $dbg;
}
#######################
##print version
sub version(){
# print STDERR "version 1.0 (29-07-2006)\n";
print STDERR "version 1.1 (31-07-2006)\n";
exit(1);
}
#usage
sub usage(){
print STDERR "qsub-wrapper.pl [options]\n";
print STDERR "Options:\n";
print STDERR "-command <file> command to run\n";
print STDERR "-stdout <file> file to save stdout of cmd (optional)\n";
print STDERR "-stderr <file> file to save stderr of cmd (optional)\n";
print STDERR "-jidfile <file> file to save the submit jobid (for submit option)\n";
print STDERR "-prevjid <id> wait for the previous job with jobid=id to finish before starting (optional)\n";
print STDERR "-qsub-prefix <string> name for sumbitted jobs (optional)\n";
print STDERR "-queue-parameters <string> parameter for the queue (optional)\n";
print STDERR "-old-sge ... assume Sun Grid Engine < 6.0\n";
print STDERR "-debug debug\n";
print STDERR "-version print version of the script\n";
print STDERR "-help this help\n";
exit(1);
}
#printparameters
sub print_parameters(){
print STDERR "command: $cmd\n";
if (defined($cmdout)){ print STDERR "file for stdout: $cmdout\n"; }
else { print STDERR "file for stdout is not defined, stdout is discarded\n"; }
if (defined($cmderr)){ print STDERR "file for stdout: $cmderr\n"; }
else { print STDERR "file for stderr is not defined, stderr is discarded\n"; }
if (defined($jidfile)){ print STDERR "file for submit job id: $jidfile\n"; }
else { print STDERR "file for submit job id is not defined, jidfile is discarded\n"; }
print STDERR "Qsub name: $qsubname\n";
print STDERR "Queue parameters: $queueparameters\n";
print STDERR "parameters directly passed to cmd: $parameters\n";
exit(1);
}
#script creation
sub preparing_script(){
my $scriptheader="\#\!/bin/bash\n# the above line is ignored by qsub, unless parameter \"-b yes\" is set!\n\n";
$scriptheader.="uname -a\n\n";
$scriptheader.="cd $workingdir\n\n";
open (OUT, "> $jobscript");
print OUT $scriptheader;
print OUT "if $cmd $parameters > $tmpdir/cmdout$uid 2> $tmpdir/cmderr$uid ; then
echo 'succeeded'
else
echo failed with exit status \$\?
die=1
fi
";
if (defined $cmdout){
print OUT "mv -f $tmpdir/cmdout$uid $cmdout || echo failed to preserve the log: $tmpdir/cmdout$uid\n\n";
}
else{
print OUT "rm -f $tmpdir/cmdout$uid\n\n";
}
if (defined $cmderr){
print OUT "mv -f $tmpdir/cmderr$uid $cmderr || echo failed to preserve the log: $tmpdir/cmderr$uid\n\n";
}
else{
print OUT "rm -f $tmpdir/cmderr$uid\n\n";
}
print OUT "if [ x\$die == 1 ]; then exit 1; fi\n";
close(OUT);
#setting permissions of the script
chmod(oct(755),$jobscript);
}
#######################
#Script starts here
init();
usage() if $cmd eq "";
safesystem("mkdir -p $tmpdir") or die;
preparing_script();
# my $maysync = $old_sge ? "" : "-sync y";
# never run in syn mode
my $maysync = "";
my $qsubcmd = "";
# create the qsubcmd to submit to the queue with the parameter "-b yes"
#my $qsubcmd="qsub $queueparameters $maysync -V -o $qsubout -e $qsuberr -N $qsubname -b yes $jobscript > $jobscript.log 2>&1";
# add -b yes if not yet defined, otherwise leave empty
$queueparameters .= " -b yes " if (index($queueparameters," -b ")==-1);
if (defined $prevjid && $prevjid!=-1 && $prevjidarraysize == 1) {
$qsubcmd="qsub $queueparameters $maysync -V -hold_jid $prevjid -o $qsubout -e $qsuberr -N $qsubname $jobscript > $jobscript.log 2>&1";
} elsif (defined $prevjid && $prevjidarraysize > 1) {
my $hj = "-hold_jid " . join(" -hold_jid ", @prevjidarray);
# print STDERR "hj is $hj\n";
$qsubcmd="qsub $queueparameters $maysync -V $hj -o $qsubout -e $qsuberr -N $qsubname $jobscript > $jobscript.log 2>&1";
} else {
$qsubcmd="qsub $queueparameters $maysync -V -o $qsubout -e $qsuberr -N $qsubname $jobscript > $jobscript.log 2>&1";
}
print "submitting $qsubcmd\n";
#run the qsubcmd
safesystem($qsubcmd) or die;
#getting id of submitted job#############
my $res;
open (IN,"$jobscript.log") or die "Can't read main job id: $jobscript.log";
chomp($res=<IN>);
my @arrayStr = split(/\s+/,$res);
my $id=$arrayStr[2];
die "Failed to get job id from $jobscript.log, got: $res"
if $id !~ /^[0-9]+$/;
close(IN);
#########################################
print STDERR " res:$res\n";
print STDERR " id:$id\n";
open (JIDOUT,">$jidfile") or die "Can't open jid file to write";
print JIDOUT "$id\n";
close(JIDOUT);
open (PIDOUT,">$jidfile.pid") or die "Can't open id.pid file to write";
print PIDOUT "$uid\n";
close(PIDOUT);
if ($old_sge) {
# need to workaround -sync, add another job that will wait for the main one
# prepare a fake waiting script
my $syncscript = "$jobscript.sync_workaround_script.sh";
safesystem("echo 'date' > $syncscript") or die;
my $checkpointfile = "$jobscript.sync_workaround_checkpoint";
# ensure checkpoint does not exist
safesystem("\\rm -f $checkpointfile") or die;
# start the 'hold' job, i.e. the job that will wait
# $cmd="qsub -cwd $queueparameters -hold_jid $id -o $checkpointfile -e /dev/null -N $qsubname.W $syncscript >& $qsubname.W.log";
$cmd="qsub -cwd $queueparameters -hold_jid $id -o $checkpointfile -e /dev/null -N $qsubname.W $syncscript >& $qsubname.W.log";
safesystem($cmd) or die;
# and wait for checkpoint file to appear
my $nr=0;
while (!-e $checkpointfile) {
sleep(10);
$nr++;
print STDERR "w" if $nr % 3 == 0;
}
safesystem("\\rm -f $checkpointfile $syncscript") or die();
print STDERR "End of waiting workaround.\n";
}
# my $failure=&check_exit_status();
# print STDERR "check_exit_status returned $failure\n";
# &kill_all_and_quit() if $failure;
# &remove_temporary_files() if !$dbg;
sub check_exit_status(){
my $failure=0;
print STDERR "check_exit_status of submitted job $id\n";
open(IN,"$qsubout") or die "Can't read $qsubout";
while (<IN>){
$failure=1 if (/failed with exit status/);
}
close(IN);
return $failure;
}
sub kill_all_and_quit(){
print STDERR "kill_all_and_quit\n";
print STDERR "qdel $id\n";
safesystem("qdel $id");
print STDERR "The submitted jobs died not correctly\n";
print STDERR "Send qdel signal to the submitted jobs\n";
exit(1);
}
sub remove_temporary_files(){
#removing temporary files
unlink("${jobscript}");
unlink("${jobscript}.log");
unlink("$qsubout");
unlink("$qsuberr");
rmdir("$tmpdir");
}
sub safesystem {
print STDERR "Executing: @_\n";
system(@_);
if ($? == -1) {
print STDERR "Failed to execute: @_\n $!\n";
exit(1);
}
elsif ($? & 127) {
printf STDERR "Execution of: @_\n died with signal %d, %s coredump\n",
($? & 127), ($? & 128) ? 'with' : 'without';
exit(1);
}
else {
my $exitcode = $? >> 8;
print STDERR "Exit code: $exitcode\n" if $exitcode;
return ! $exitcode;
}
}
# look for the correct pwdcmd (pwd by default, pawd if it exists)
# I assume that pwd always exists
sub getPwdCmd(){
my $pwdcmd="pwd";
my $a;
chomp($a=`which pawd | head -1 | awk '{print $1}'`);
if ($a && -e $a){ $pwdcmd=$a; }
return $pwdcmd;
}

1633
contrib/mert-sge-nosync/training/mert-moses-sge-nosync.pl Executable file
View File

File diff suppressed because it is too large Load Diff

64
contrib/mert-sge-nosync/training/sge-nosync/cleartmpfiles.pl Executable file
View File

@ -0,0 +1,64 @@
#!/usr/bin/perl
my @filename_id = "";
my $this_id = "";
# remove exitjob and forceexitjob
chomp(my @rddfile_list = `ls exitjob* forceexitjob*`);
foreach my $rddfile (@rddfile_list) {
unlink("$rddfile");
}
chomp(@filename_id = `ls *.id | grep -v 'clear'`);
open (OUT, "> all.id.all");
print OUT "==Combine log at ".`date`;
print OUT `tail -n +1 *.id`;
print OUT "==LOG combined ".`date`;
close(OUT);
foreach $this_id (@filename_id) {
# print OUT `cat $this_id`;
unlink("$this_id");
}
chomp (@filename_id = `ls *.id.pid | grep -v 'clear'`);
open (OUT, "> all.id.pid.all");
print OUT "==Combine log at ".`date`;
print OUT `tail -n +1 *.id.pid`;
print OUT "==Log combined ".`date`;
close(OUT);
foreach $this_id (@filename_id) {
# print OUT `cat $this_id`;
unlink("$this_id");
}
chomp(@filename_id = `ls *.out | grep -v 'clear'`);
open (OUT, "> all.out.all");
print OUT "==Combine log at ".`date`;
print OUT `tail -n +1 *.out`;
print OUT "==Log combined ".`date`;
close(OUT);
foreach $this_id (@filename_id) {
# print OUT `cat $this_id`;
unlink("$this_id");
}
chomp(@filename_id = `ls *.err | grep -v 'clear'`);
open (OUT, "> all.err.all");
print OUT "==Combine log at ".`date`;
print OUT `tail -n +1 *.err`;
print OUT "==Log combined ".`date`;
close(OUT);
foreach $this_id (@filename_id) {
# print OUT `cat $this_id`;
unlink("$this_id");
}
# waitall.sh which cannot be deleted inside moses-parallel-sge-nosync.pl
chomp(@filename_id = `ls *waitall.sh`);
foreach $this_id (@filename_id) {
unlink("$this_id");
}

426
contrib/mert-sge-nosync/training/sge-nosync/create-config-sge-nosync.pl Executable file
View File

@ -0,0 +1,426 @@
#!/usr/bin/perl -w
# $Id$
# after filter-mode-given-input.pl, process the feature list
# original code by Philipp Koehn
# changes by Ondrej Bojar
# adapted for hierarchical models by Phil Williams
use strict;
use FindBin qw($Bin);
use Getopt::Long;
my $SCRIPTS_ROOTDIR;
if (defined($ENV{"SCRIPTS_ROOTDIR"})) {
$SCRIPTS_ROOTDIR = $ENV{"SCRIPTS_ROOTDIR"};
} else {
$SCRIPTS_ROOTDIR = $Bin;
if ($SCRIPTS_ROOTDIR eq '') {
$SCRIPTS_ROOTDIR = dirname(__FILE__);
}
$SCRIPTS_ROOTDIR =~ s/\/training$//;
$ENV{"SCRIPTS_ROOTDIR"} = $SCRIPTS_ROOTDIR;
}
# moses.ini file uses FULL names for lambdas, while this training script
# internally (and on the command line) uses ABBR names.
my @ABBR_FULL_MAP = qw(d=weight-d lm=weight-l tm=weight-t w=weight-w
g=weight-generation lex=weight-lex I=weight-i);
my %ABBR2FULL = map {split/=/,$_,2} @ABBR_FULL_MAP;
my %FULL2ABBR = map {my ($a, $b) = split/=/,$_,2; ($b, $a);} @ABBR_FULL_MAP;
my $verbose = 0;
my $usage = 0; # request for --help
##!# # consider phrases in input up to $MAX_LENGTH
##!# # in other words, all phrase-tables will be truncated at least to 10 words per
##!# # phrase.
##!# my $MAX_LENGTH = 10;
# utilities
##!# my $ZCAT = "gzip -cd";
# get optional parameters
##!# my $opt_hierarchical = 0;
##!# my $binarizer = undef;
##!# my $opt_min_non_initial_rule_count = undef;
##!# my $opt_gzip = 1; # gzip output files (so far only phrase-based ttable until someone tests remaining models and formats)
my $___RANGES = undef;
my $___ACTIVATE_FEATURES = undef; # comma-separated (or blank-separated) list of features to work on
# if undef work on all features
# (others are fixed to the starting values)
my $___DECODER_FLAGS = ""; # additional parametrs to pass to the decoder
my $devbleu = undef;
my $___WORKING_DIR = undef;
my $___DEV_F = undef;
my $run = undef; # either first or final
my $runid_final = undef;
my $runid_finalplus=0;
my $sparse_weights_file = undef;
# set 0 if input type is text, set 1 if input type is confusion network
my $___INPUTTYPE = 0;
my $___DECODER = undef; # required, pathname to the decoder executable
my $___CONFIG = undef; # required, pathname to startup ini file
GetOptions(
"activate-features=s" => \$___ACTIVATE_FEATURES, #comma-separated (or blank-separated) list of features to work on (others are fixed to the starting values)
"range=s@" => \$___RANGES,
"decoder-flags=s" => \$___DECODER_FLAGS,
"inputtype=i" => \$___INPUTTYPE,
"devbleu=s" => \$devbleu,
"sparse_weight_file=s" => \$sparse_weights_file,
"working-dir=s" => \$___WORKING_DIR,
) or exit(1);
##!# GetOptions(
##!# "gzip!" => \$opt_gzip,
##!# "Hierarchical" => \$opt_hierarchical,
##!# "Binarizer=s" => \$binarizer,
##!# "MinNonInitialRuleCount=i" => \$opt_min_non_initial_rule_count
##!# ) or exit(1);
# the ?? required parameters can be supplied on the command line directly
# or using the --options
if (scalar @ARGV == 4) {
# required parameters: options
$___DEV_F = shift;
$___DECODER = shift;
$___CONFIG = shift;
$run = shift; # first or final
}
if ($usage || !defined $___DECODER || !defined $___CONFIG) {
print STDERR "usage: $0 \$___DECODER \$___CONFIG(decoder.ini)
Options:
--activate-features=STRING ... comma-separated list of features to optimize,
others are fixed to the starting values
default: optimize all features
example: tm_0,tm_4,d_0
--range=tm:0..1,-1..1 ... specify min and max value for some features
--range can be repeated as needed.
The order of the various --range specifications
is important only within a feature name.
E.g.:
--range=tm:0..1,-1..1 --range=tm:0..2
is identical to:
--range=tm:0..1,-1..1,0..2
but not to:
--range=tm:0..2 --range=tm:0..1,-1..1
--decoder-flags=STRING ... extra parameters for the decoder
--inputtype=[0|1|2] ... Handle different input types: (0 for text,
1 for confusion network, 2 for lattices,
default is 0)
";
exit 1;
}
##!# # get command line parameters
##!# my $dir = shift;
##!# my $config = shift;
##!# my $input = shift;
##!# $dir = ensure_full_path($dir);
############################################################
############################################################
############################################################
# main
# we run moses to check validity of moses.ini and to obtain all the feature
# names
if (($run eq "first")){
my $featlist = get_featlist_from_moses($___CONFIG,$___CONFIG,"first");
$featlist = insert_ranges_to_featlist($featlist, $___RANGES);
create_config($___CONFIG,"$___WORKING_DIR/run1.moses.ini",$featlist,1,(defined$devbleu?$devbleu:"--not-estimated--"),$sparse_weights_file);
} else { # $run eq "final"
chomp ($runid_final = `cat $___WORKING_DIR/finished_step.txt | tail -n 1`);
$runid_finalplus = $runid_final + 1;
`mv run${runid_finalplus}.moses.ini run_final.moses.ini`;
chomp ($devbleu = `cat $___WORKING_DIR/run_final.moses.ini | tail -n +3 | head -n 1 | gawk '{print \$3}'`);
my $featlist = get_featlist_from_moses($___CONFIG,"$___WORKING_DIR/run_final.moses.ini","final");
$featlist = insert_ranges_to_featlist($featlist, $___RANGES);
create_config($___CONFIG,"$___WORKING_DIR/moses.ini",$featlist,$runid_finalplus,$devbleu,$sparse_weights_file);
}
##COPIED## Mark which features are disabled:
##COPIED#if (defined $___ACTIVATE_FEATURES) {
##COPIED# my %enabled = map { ($_, 1) } split /[, ]+/, $___ACTIVATE_FEATURES;
##COPIED# my %cnt;
##COPIED# for(my $i=0; $i<scalar(@{$featlist->{"names"}}); $i++) {
##COPIED# my $name = $featlist->{"names"}->[$i];
##COPIED# $cnt{$name} = 0 if !defined $cnt{$name};
##COPIED# $featlist->{"enabled"}->[$i] = $enabled{$name."_".$cnt{$name}};
##COPIED# $cnt{$name}++;
##COPIED# }
##COPIED#} else {
##COPIED# # all enabled
##COPIED# for(my $i=0; $i<scalar(@{$featlist->{"names"}}); $i++) {
##COPIED# $featlist->{"enabled"}->[$i] = 1;
##COPIED# }
##COPIED#}
##COPIED#
##COPIED#print STDERR "MERT starting values and ranges for random generation:\n";
##COPIED#for(my $i=0; $i<scalar(@{$featlist->{"names"}}); $i++) {
##COPIED# my $name = $featlist->{"names"}->[$i];
##COPIED# my $val = $featlist->{"values"}->[$i];
##COPIED# my $min = $featlist->{"mins"}->[$i];
##COPIED# my $max = $featlist->{"maxs"}->[$i];
##COPIED# my $enabled = $featlist->{"enabled"}->[$i];
##COPIED# printf STDERR " %5s = %7.3f", $name, $val;
##COPIED# if ($enabled) {
##COPIED# printf STDERR " (%5.2f .. %5.2f)\n", $min, $max;
##COPIED# } else {
##COPIED# print STDERR " --- inactive, not optimized ---\n";
##COPIED# }
##COPIED#}
sub get_featlist_from_moses {
# run moses with the given config file and return the list of features and
# their initial values
my $configfn = shift;
my $config_score = shift;
my $run = shift;
my $featlistfn = "";
if ($run eq 'first') {
$featlistfn = "./features.list"; # given feature list
} elsif ($run eq "final") {
$featlistfn = "./features.list.run_final";
}
if (-e $featlistfn) {
print STDERR "Using cached features list: $featlistfn\n";
} else {
print STDERR "Asking moses for feature names and values from $config_score\n";
my $cmd = "$___DECODER $___DECODER_FLAGS -config $config_score -inputtype $___INPUTTYPE -show-weights > $featlistfn";
print STDERR "$cmd\n"; #DEBUG
safesystem($cmd) or die "Failed to run moses with the config $config_score";
}
# read feature list
my @names = ();
my @startvalues = ();
open(INI,$featlistfn) or die "Can't read $featlistfn";
my $nr = 0;
my @errs = ();
while (<INI>) {
$nr++;
chomp;
/^(.+) (\S+) (\S+)$/ || die("invalid feature: $_");
my ($longname, $feature, $value) = ($1,$2,$3);
next if $value eq "sparse";
push @errs, "$featlistfn:$nr:Bad initial value of $feature: $value\n"
if $value !~ /^[+-]?[0-9.e]+$/;
push @errs, "$featlistfn:$nr:Unknown feature '$feature', please add it to \@ABBR_FULL_MAP\n"
if !defined $ABBR2FULL{$feature};
push @names, $feature;
push @startvalues, $value;
}
close INI;
if (scalar @errs) {
print STDERR join("", @errs);
exit 1;
}
return {"names"=>\@names, "values"=>\@startvalues};
}
sub insert_ranges_to_featlist {
my $featlist = shift;
my $ranges = shift;
$ranges = [] if !defined $ranges;
# first collect the ranges from options
my $niceranges;
foreach my $range (@$ranges) {
my $name = undef;
foreach my $namedpair (split /,/, $range) {
if ($namedpair =~ /^(.*?):/) {
$name = $1;
$namedpair =~ s/^.*?://;
die "Unrecognized name '$name' in --range=$range"
if !defined $ABBR2FULL{$name};
}
my ($min, $max) = split /\.\./, $namedpair;
die "Bad min '$min' in --range=$range" if $min !~ /^-?[0-9.]+$/;
die "Bad max '$max' in --range=$range" if $min !~ /^-?[0-9.]+$/;
die "No name given in --range=$range" if !defined $name;
push @{$niceranges->{$name}}, [$min, $max];
}
}
# now populate featlist
my $seen = undef;
for(my $i=0; $i<scalar(@{$featlist->{"names"}}); $i++) {
my $name = $featlist->{"names"}->[$i];
$seen->{$name} ++;
my $min = 0.0;
my $max = 1.0;
if (defined $niceranges->{$name}) {
my $minmax = shift @{$niceranges->{$name}};
($min, $max) = @$minmax if defined $minmax;
}
$featlist->{"mins"}->[$i] = $min;
$featlist->{"maxs"}->[$i] = $max;
}
return $featlist;
}
sub safesystem {
print STDERR "Executing: @_\n";
system(@_);
if ($? == -1) {
print STDERR "Failed to execute: @_\n $!\n";
exit(1);
}
elsif ($? & 127) {
printf STDERR "Execution of: @_\n died with signal %d, %s coredump\n",
($? & 127), ($? & 128) ? 'with' : 'without';
exit(1);
}
else {
my $exitcode = $? >> 8;
print STDERR "Exit code: $exitcode\n" if $exitcode;
return ! $exitcode;
}
}
sub create_config {
my $infn = shift; # source config
my $outfn = shift; # where to save the config
my $featlist = shift; # the lambdas we should write
my $iteration = shift; # just for verbosity
my $bleu_achieved = shift; # just for verbosity
my $sparse_weights_file = shift; # only defined when optimizing sparse features
my %P; # the hash of all parameters we wish to override
# first convert the command line parameters to the hash
{ # ensure local scope of vars
my $parameter=undef;
print "Parsing --decoder-flags: |$___DECODER_FLAGS|\n";
$___DECODER_FLAGS =~ s/^\s*|\s*$//;
$___DECODER_FLAGS =~ s/\s+/ /;
foreach (split(/ /,$___DECODER_FLAGS)) {
if (/^\-([^\d].*)$/) {
$parameter = $1;
$parameter = $ABBR2FULL{$parameter} if defined($ABBR2FULL{$parameter});
}
else {
die "Found value with no -paramname before it: $_"
if !defined $parameter;
push @{$P{$parameter}},$_;
}
}
}
# First delete all weights params from the input, we're overwriting them.
# Delete both short and long-named version.
for(my $i=0; $i<scalar(@{$featlist->{"names"}}); $i++) {
my $name = $featlist->{"names"}->[$i];
delete($P{$name});
delete($P{$ABBR2FULL{$name}});
}
# Convert weights to elements in P
for(my $i=0; $i<scalar(@{$featlist->{"names"}}); $i++) {
my $name = $featlist->{"names"}->[$i];
my $val = $featlist->{"values"}->[$i];
$name = defined $ABBR2FULL{$name} ? $ABBR2FULL{$name} : $name;
# ensure long name
push @{$P{$name}}, $val;
}
if (defined($sparse_weights_file)) {
push @{$P{"weights-file"}}, $___WORKING_DIR."/".$sparse_weights_file;
}
# create new moses.ini decoder config file by cloning and overriding the original one
open(INI,$infn) or die "Can't read $infn";
delete($P{"config"}); # never output
print "Saving new config to: $outfn\n";
open(OUT,"> $outfn") or die "Can't write $outfn";
print OUT "# MERT optimized configuration\n";
print OUT "# decoder $___DECODER\n";
print OUT "# BLEU $bleu_achieved on dev $___DEV_F\n";
print OUT "# We were before running iteration $iteration\n";
print OUT "# finished ".`date`;
my $line = <INI>;
while(1) {
last unless $line;
# skip until hit [parameter]
if ($line !~ /^\[(.+)\]\s*$/) {
$line = <INI>;
print OUT $line if $line =~ /^\#/ || $line =~ /^\s+$/;
next;
}
# parameter name
my $parameter = $1;
$parameter = $ABBR2FULL{$parameter} if defined($ABBR2FULL{$parameter});
print OUT "[$parameter]\n";
# change parameter, if new values
if (defined($P{$parameter})) {
# write new values
foreach (@{$P{$parameter}}) {
print OUT $_."\n";
}
delete($P{$parameter});
# skip until new parameter, only write comments
while($line = <INI>) {
print OUT $line if $line =~ /^\#/ || $line =~ /^\s+$/;
last if $line =~ /^\[/;
last unless $line;
}
next;
}
# unchanged parameter, write old
while($line = <INI>) {
last if $line =~ /^\[/;
print OUT $line;
}
}
# write all additional parameters
foreach my $parameter (keys %P) {
print OUT "\n[$parameter]\n";
foreach (@{$P{$parameter}}) {
print OUT $_."\n";
}
}
close(INI);
close(OUT);
print STDERR "Saved: $outfn\n";
}

235
contrib/mert-sge-nosync/training/sge-nosync/moses-parallel-postdecode-sge-nosync.pl Executable file
View File

@ -0,0 +1,235 @@
#!/usr/bin/perl
my $logflag="";
my $logfile="";
my $alifile=undef;
my $nbestflag=0;
my $processid=0;
my $idxliststr="";
my $workingdir="";
my $inputfile="";
my $tmpdir="";
my $splitpfx="";
my $jobscript="";
my $qsubout="";
my $qsuberr="";
my $nbestfile=undef;
my $nbestlist=undef;
my $outnbest="";
my $lsamp_filename="";
my @idxlist=();
###############################
# Script starts here
init();
#concatenating translations and removing temporary files
concatenate_1best();
concatenate_logs() if $logflag;
concatenate_ali() if defined $alifile;
concatenate_nbest() if $nbestflag;
safesystem("cat nbest$$ >> /dev/stdout") if $nbestlist[0] eq '-';
print STDERR "Not support searchgraphflag for sync mert\n" if $searchgraphflag;
# concatenate_searchgraph() if $searchgraphflag;
# safesystem("cat searchgraph$$ >> /dev/stdout") if $searchgraphlist eq '-';
print STDERR "Not support wordgraphflag for sync mert\n" if $searchgraphflag;
# concatenate_wordgraph() if $wordgraphflag;
# safesystem("cat wordgraph$$ >> /dev/stdout") if $wordgraphlist[0] eq '-';
remove_temporary_files();
####
#### ### ending scripts in run_decoder() ##############
#### sanity_check_order_of_lambdas($featlist, $filename);
#### ## how to do return???
#### return ($filename, $lsamp_filename);
######################################################
sub init(){
use strict;
use Getopt::Long qw(:config pass_through no_ignore_case permute);
GetOptions('alignment-output-file=s'=>\$alifile,
'process-id=s'=>\$processid,
'idxliststr=s'=>\$idxliststr,
'logfile=s'=>\$logfile,
'nbestfile=s'=>\$nbestfile,
'outnbest=s'=>\$outnbest,
'lsamp-filename=s'=>\$lsamp_filename,
'input-file=s'=>\$inputfile
) or exit(1);
if ($logfile){ $logflag=1; }
if (defined $nbestfile) { $nbestflag=1; }
$idxliststr =~ s/^\s+|\s+$//g;
@idxlist = split(/\s+/,$idxliststr);
my $pwdcmd = getPwdCmd();
$workingdir = `$pwdcmd`; chomp $workingdir;
$tmpdir="$workingdir/tmp$processid";
$splitpfx="split$processid";
$jobscript="$workingdir/job$processid";
$qsubout="$workingdir/out.job$processid";
$qsuberr="$workingdir/err.job$processid";
# print STDERR "$idxliststr\n";
}
sub concatenate_nbest(){
my $oldcode="";
my $newcode=-1;
my %inplength = ();
my $offset = 0;
# get the list of feature and set a fictitious string with zero scores
open (IN, "${nbestfile}.${splitpfx}$idxlist[0]");
my $str = <IN>;
chomp($str);
close(IN);
my ($code,$trans,$featurescores,$globalscore)=split(/\|\|\|/,$str);
my $emptytrans = " ";
my $emptyglobalscore = " 0.0";
my $emptyfeaturescores = $featurescores;
$emptyfeaturescores =~ s/[-0-9\.]+/0/g;
if ($outnbest eq '-'){ $outnbest="nbest$processid"; }
# my $outnbest=$nbestlist[0];
# if ($nbestlist[0] eq '-'){ $outnbest="nbest$$"; }
open (OUT, "> $outnbest");
foreach my $idx (@idxlist){
#computing the length of each input file
# print STDERR "this idx: $idx\n";
my @in=();
open (IN, "${inputfile}.${splitpfx}${idx}.trans");
@in=<IN>;
close(IN);
$inplength{$idx} = scalar(@in);
open (IN, "${nbestfile}.${splitpfx}${idx}");
while (<IN>){
my ($code,@extra)=split(/\|\|\|/,$_);
$code += $offset;
if ($code ne $oldcode){
# if there is a jump between two consecutive codes
# it means that an input sentence is not translated
# fill this hole with a "fictitious" list of translation
# comprising just one "emtpy translation" with zero scores
while ($code - $oldcode > 1){
$oldcode++;
print OUT join("\|\|\|",($oldcode,$emptytrans,$emptyfeaturescores,$emptyglobalscore)),"\n";
}
}
$oldcode=$code;
print OUT join("\|\|\|",($oldcode,@extra));
}
close(IN);
$offset += $inplength{$idx};
while ($offset - $oldcode > 1){
$oldcode++;
print OUT join("\|\|\|",($oldcode,$emptytrans,$emptyfeaturescores,$emptyglobalscore)),"\n";
}
}
close(OUT);
}
sub concatenate_1best(){
foreach my $idx (@idxlist){
# print STDERR "reading 1best file ${inputfile}.${splitpfx}$idx.trans\n";
my @in=();
open (IN, "${inputfile}.${splitpfx}${idx}.trans");
@in=<IN>;
# print STDERR "in array is : @in";
print STDOUT "@in";
close(IN);
}
}
sub concatenate_logs(){
open (OUT, "> ${logfile}");
foreach my $idx (@idxlist){
my @in=();
open (IN, "$qsubout$idx");
@in=<IN>;
print OUT "@in";
close(IN);
}
close(OUT);
}
sub concatenate_ali(){
open (OUT, "> ${alifile}");
foreach my $idx (@idxlist){
my @in=();
open (IN, "$alifile.$splitpfx$idx");
@in=<IN>;
print OUT "@in";
close(IN);
}
close(OUT);
}
# look for the correct pwdcmd (pwd by default, pawd if it exists)
# I assume that pwd always exists
sub getPwdCmd(){
my $pwdcmd="pwd";
my $a;
chomp($a=`which pawd | head -1 | awk '{print $1}'`);
if ($a && -e $a){ $pwdcmd=$a; }
return $pwdcmd;
}
sub remove_temporary_files(){
# removing temporary files
foreach my $idx (@idxlist){
unlink("${inputfile}.${splitpfx}${idx}.trans");
unlink("${inputfile}.${splitpfx}${idx}");
if (defined $alifile){ unlink("${alifile}.${splitpfx}${idx}"); }
if ($nbestflag){ unlink("${nbestfile}.${splitpfx}${idx}"); }
if ($searchgraphflag){ unlink("${searchgraphfile}.${splitpfx}${idx}"); }
if ($wordgraphflag){ unlink("${wordgraphfile}.${splitpfx}${idx}"); }
# print STDERR "Deleting ${jobscript}${idx}.bash\n";
unlink("${jobscript}${idx}.bash");
unlink("${jobscript}${idx}.log");
unlink("$qsubname.W.log");
unlink("$qsubout$idx");
unlink("$qsuberr$idx");
rmdir("$tmpdir");
}
# unlink("${jobscript}.sync_workaround_script.sh");
if ($nbestflag && $nbestlist[0] eq '-'){ unlink("${nbestfile}$$"); };
if ($searchgraphflag && $searchgraphlist eq '-'){ unlink("${searchgraphfile}$$"); };
if ($wordgraphflag && $wordgraphlist eq '-'){ unlink("${wordgraphfile}$$"); };
}

30
contrib/mert-sge-nosync/training/sge-nosync/poll-decoder.pl Executable file
View File

@ -0,0 +1,30 @@
#!/usr/bin/perl
use Getopt::Long qw(:config pass_through no_ignore_case permute);
my $poll_target = undef;
my $working_dir = undef;
GetOptions('poll-target=s'=> \$poll_target,
'working-dir'=> \$working_dir
) or exit(1);
if (defined $working_dir) {
chdir($working_dir);
}
my $cnt = 1;
print STDERR "Wait for file: $poll_target\n";
while (1) {
if (-e $poll_target){
print STDERR "\n File found!!\n";
last;
} else {
sleep(10);
print STDERR ".";
}
}

283
contrib/mert-sge-nosync/training/sge-nosync/process-featlist-sge-nosync.pl Executable file
View File

@ -0,0 +1,283 @@
#!/usr/bin/perl -w
# $Id$
# after filter-mode-given-input.pl, process the feature list
# original code by Philipp Koehn
# changes by Ondrej Bojar
# adapted for hierarchical models by Phil Williams
use strict;
use FindBin qw($Bin);
use Getopt::Long;
my $SCRIPTS_ROOTDIR;
if (defined($ENV{"SCRIPTS_ROOTDIR"})) {
$SCRIPTS_ROOTDIR = $ENV{"SCRIPTS_ROOTDIR"};
} else {
$SCRIPTS_ROOTDIR = $Bin;
if ($SCRIPTS_ROOTDIR eq '') {
$SCRIPTS_ROOTDIR = dirname(__FILE__);
}
$SCRIPTS_ROOTDIR =~ s/\/training$//;
$ENV{"SCRIPTS_ROOTDIR"} = $SCRIPTS_ROOTDIR;
}
# moses.ini file uses FULL names for lambdas, while this training script
# internally (and on the command line) uses ABBR names.
my @ABBR_FULL_MAP = qw(d=weight-d lm=weight-l tm=weight-t w=weight-w
g=weight-generation lex=weight-lex I=weight-i);
my %ABBR2FULL = map {split/=/,$_,2} @ABBR_FULL_MAP;
my %FULL2ABBR = map {my ($a, $b) = split/=/,$_,2; ($b, $a);} @ABBR_FULL_MAP;
my $verbose = 0;
my $usage = 0; # request for --help
##!# # consider phrases in input up to $MAX_LENGTH
##!# # in other words, all phrase-tables will be truncated at least to 10 words per
##!# # phrase.
##!# my $MAX_LENGTH = 10;
# utilities
##!# my $ZCAT = "gzip -cd";
# get optional parameters
##!# my $opt_hierarchical = 0;
##!# my $binarizer = undef;
##!# my $opt_min_non_initial_rule_count = undef;
##!# my $opt_gzip = 1; # gzip output files (so far only phrase-based ttable until someone tests remaining models and formats)
my $___RANGES = undef;
my $___ACTIVATE_FEATURES = undef; # comma-separated (or blank-separated) list of features to work on
# if undef work on all features
# (others are fixed to the starting values)
my $___DECODER_FLAGS = ""; # additional parametrs to pass to the decoder
# set 0 if input type is text, set 1 if input type is confusion network
my $___INPUTTYPE = 0;
my $___DECODER = undef; # required, pathname to the decoder executable
my $___CONFIG = undef; # required, pathname to startup ini file
GetOptions(
"activate-features=s" => \$___ACTIVATE_FEATURES, #comma-separated (or blank-separated) list of features to work on (others are fixed to the starting values)
"range=s@" => \$___RANGES,
"decoder-flags=s" => \$___DECODER_FLAGS,
"inputtype=i" => \$___INPUTTYPE
) or exit(1);
##!# GetOptions(
##!# "gzip!" => \$opt_gzip,
##!# "Hierarchical" => \$opt_hierarchical,
##!# "Binarizer=s" => \$binarizer,
##!# "MinNonInitialRuleCount=i" => \$opt_min_non_initial_rule_count
##!# ) or exit(1);
# the ?? required parameters can be supplied on the command line directly
# or using the --options
if (scalar @ARGV == 2) {
# required parameters: options
$___DECODER = shift;
$___CONFIG = shift;
}
if ($usage || !defined $___DECODER || !defined $___CONFIG) {
print STDERR "usage: $0 \$___DECODER \$___CONFIG(decoder.ini)
Options:
--activate-features=STRING ... comma-separated list of features to optimize,
others are fixed to the starting values
default: optimize all features
example: tm_0,tm_4,d_0
--range=tm:0..1,-1..1 ... specify min and max value for some features
--range can be repeated as needed.
The order of the various --range specifications
is important only within a feature name.
E.g.:
--range=tm:0..1,-1..1 --range=tm:0..2
is identical to:
--range=tm:0..1,-1..1,0..2
but not to:
--range=tm:0..2 --range=tm:0..1,-1..1
--decoder-flags=STRING ... extra parameters for the decoder
--inputtype=[0|1|2] ... Handle different input types: (0 for text,
1 for confusion network, 2 for lattices,
default is 0)
";
exit 1;
}
##!# # get command line parameters
##!# my $dir = shift;
##!# my $config = shift;
##!# my $input = shift;
##!# $dir = ensure_full_path($dir);
############################################################
############################################################
############################################################
# main
# we run moses to check validity of moses.ini and to obtain all the feature
# names
my $featlist = get_featlist_from_moses($___CONFIG);
$featlist = insert_ranges_to_featlist($featlist, $___RANGES);
# Mark which features are disabled:
if (defined $___ACTIVATE_FEATURES) {
my %enabled = map { ($_, 1) } split /[, ]+/, $___ACTIVATE_FEATURES;
my %cnt;
for(my $i=0; $i<scalar(@{$featlist->{"names"}}); $i++) {
my $name = $featlist->{"names"}->[$i];
$cnt{$name} = 0 if !defined $cnt{$name};
$featlist->{"enabled"}->[$i] = $enabled{$name."_".$cnt{$name}};
$cnt{$name}++;
}
} else {
# all enabled
for(my $i=0; $i<scalar(@{$featlist->{"names"}}); $i++) {
$featlist->{"enabled"}->[$i] = 1;
}
}
print STDERR "MERT starting values and ranges for random generation:\n";
for(my $i=0; $i<scalar(@{$featlist->{"names"}}); $i++) {
my $name = $featlist->{"names"}->[$i];
my $val = $featlist->{"values"}->[$i];
my $min = $featlist->{"mins"}->[$i];
my $max = $featlist->{"maxs"}->[$i];
my $enabled = $featlist->{"enabled"}->[$i];
printf STDERR " %5s = %7.3f", $name, $val;
if ($enabled) {
printf STDERR " (%5.2f .. %5.2f)\n", $min, $max;
} else {
print STDERR " --- inactive, not optimized ---\n";
}
}
sub get_featlist_from_moses {
# run moses with the given config file and return the list of features and
# their initial values
my $configfn = shift;
my $featlistfn = "./features.list";
if (-e $featlistfn) {
print STDERR "Using cached features list: $featlistfn\n";
} else {
print STDERR "Asking moses for feature names and values from $___CONFIG\n";
my $cmd = "$___DECODER $___DECODER_FLAGS -config $configfn -inputtype $___INPUTTYPE -show-weights > $featlistfn";
print STDERR "$cmd\n"; #DEBUG
safesystem($cmd) or die "Failed to run moses with the config $configfn";
}
# read feature list
my @names = ();
my @startvalues = ();
open(INI,$featlistfn) or die "Can't read $featlistfn";
my $nr = 0;
my @errs = ();
while (<INI>) {
$nr++;
chomp;
/^(.+) (\S+) (\S+)$/ || die("invalid feature: $_");
my ($longname, $feature, $value) = ($1,$2,$3);
next if $value eq "sparse";
push @errs, "$featlistfn:$nr:Bad initial value of $feature: $value\n"
if $value !~ /^[+-]?[0-9.e]+$/;
push @errs, "$featlistfn:$nr:Unknown feature '$feature', please add it to \@ABBR_FULL_MAP\n"
if !defined $ABBR2FULL{$feature};
push @names, $feature;
push @startvalues, $value;
}
close INI;
if (scalar @errs) {
print STDERR join("", @errs);
exit 1;
}
return {"names"=>\@names, "values"=>\@startvalues};
}
sub insert_ranges_to_featlist {
my $featlist = shift;
my $ranges = shift;
$ranges = [] if !defined $ranges;
# first collect the ranges from options
my $niceranges;
foreach my $range (@$ranges) {
my $name = undef;
foreach my $namedpair (split /,/, $range) {
if ($namedpair =~ /^(.*?):/) {
$name = $1;
$namedpair =~ s/^.*?://;
die "Unrecognized name '$name' in --range=$range"
if !defined $ABBR2FULL{$name};
}
my ($min, $max) = split /\.\./, $namedpair;
die "Bad min '$min' in --range=$range" if $min !~ /^-?[0-9.]+$/;
die "Bad max '$max' in --range=$range" if $min !~ /^-?[0-9.]+$/;
die "No name given in --range=$range" if !defined $name;
push @{$niceranges->{$name}}, [$min, $max];
}
}
# now populate featlist
my $seen = undef;
for(my $i=0; $i<scalar(@{$featlist->{"names"}}); $i++) {
my $name = $featlist->{"names"}->[$i];
$seen->{$name} ++;
my $min = 0.0;
my $max = 1.0;
if (defined $niceranges->{$name}) {
my $minmax = shift @{$niceranges->{$name}};
($min, $max) = @$minmax if defined $minmax;
}
$featlist->{"mins"}->[$i] = $min;
$featlist->{"maxs"}->[$i] = $max;
}
return $featlist;
}
sub safesystem {
print STDERR "Executing: @_\n";
system(@_);
if ($? == -1) {
print STDERR "Failed to execute: @_\n $!\n";
exit(1);
}
elsif ($? & 127) {
printf STDERR "Execution of: @_\n died with signal %d, %s coredump\n",
($? & 127), ($? & 128) ? 'with' : 'without';
exit(1);
}
else {
my $exitcode = $? >> 8;
print STDERR "Exit code: $exitcode\n" if $exitcode;
return ! $exitcode;
}
}

1377
contrib/mert-sge-nosync/training/sge-nosync/process-moses-result-sge-nosync.pl Executable file
View File

File diff suppressed because it is too large Load Diff

271
contrib/mert-sge-nosync/training/sge-nosync/run-decoder-sge-nosync.pl Executable file
View File

@ -0,0 +1,271 @@
#!/usr/bin/perl -w
# $Id$
# after filter-mode-given-input.pl, process the feature list
# original code by Philipp Koehn
# changes by Ondrej Bojar
# adapted for hierarchical models by Phil Williams
use strict;
use FindBin qw($Bin);
use Getopt::Long;
my $SCRIPTS_ROOTDIR;
if (defined($ENV{"SCRIPTS_ROOTDIR"})) {
$SCRIPTS_ROOTDIR = $ENV{"SCRIPTS_ROOTDIR"};
} else {
$SCRIPTS_ROOTDIR = $Bin;
if ($SCRIPTS_ROOTDIR eq '') {
$SCRIPTS_ROOTDIR = dirname(__FILE__);
}
$SCRIPTS_ROOTDIR =~ s/\/training$//;
$ENV{"SCRIPTS_ROOTDIR"} = $SCRIPTS_ROOTDIR;
}
##!# # moses.ini file uses FULL names for lambdas, while this training script
##!# # internally (and on the command line) uses ABBR names.
##!# my @ABBR_FULL_MAP = qw(d=weight-d lm=weight-l tm=weight-t w=weight-w
##!# g=weight-generation lex=weight-lex I=weight-i);
##!# my %ABBR2FULL = map {split/=/,$_,2} @ABBR_FULL_MAP;
##!# my %FULL2ABBR = map {my ($a, $b) = split/=/,$_,2; ($b, $a);} @ABBR_FULL_MAP;
my $verbose = 0;
my $usage = 0; # request for --help
##!# # consider phrases in input up to $MAX_LENGTH
##!# # in other words, all phrase-tables will be truncated at least to 10 words per
##!# # phrase.
##!# my $MAX_LENGTH = 10;
# utilities
##!# my $ZCAT = "gzip -cd";
# get optional parameters
##!# my $opt_hierarchical = 0;
##!# my $binarizer = undef;
##!# my $opt_min_non_initial_rule_count = undef;
##!# my $opt_gzip = 1; # gzip output files (so far only phrase-based ttable until someone tests remaining models and formats)
my $___RANGES = undef;
my $___ACTIVATE_FEATURES = undef; # comma-separated (or blank-separated) list of features to work on
# if undef work on all features
# (others are fixed to the starting values)
my $___DECODER_FLAGS = ""; # additional parametrs to pass to the decoder
# set 0 if input type is text, set 1 if input type is confusion network
my $___INPUTTYPE = 0;
my $___DECODER = undef; # required, pathname to the decoder executable
my $___CONFIG = undef; # required, pathname to startup ini file
GetOptions(
##!# "activate-features=s" => \$___ACTIVATE_FEATURES, #comma-separated (or blank-separated) list of features to work on (others are fixed to the starting values)
##!# "range=s@" => \$___RANGES,
##!# "decoder-flags=s" => \$___DECODER_FLAGS,
##!# "inputtype=i" => \$___INPUTTYPE
""
) or exit(1);
# the ?? required parameters can be supplied on the command line directly
# or using the --options
if (scalar @ARGV == 2) {
# required parameters: options
$___DECODER = shift;
$___CONFIG = shift;
}
if ($usage || !defined $___DECODER || !defined $___CONFIG) {
print STDERR "usage: $0 \$___DECODER \$___CONFIG(decoder.ini)
Options:
--activate-features=STRING ... comma-separated list of features to optimize,
others are fixed to the starting values
default: optimize all features
example: tm_0,tm_4,d_0
--range=tm:0..1,-1..1 ... specify min and max value for some features
--range can be repeated as needed.
The order of the various --range specifications
is important only within a feature name.
E.g.:
--range=tm:0..1,-1..1 --range=tm:0..2
is identical to:
--range=tm:0..1,-1..1,0..2
but not to:
--range=tm:0..2 --range=tm:0..1,-1..1
--decoder-flags=STRING ... extra parameters for the decoder
--inputtype=[0|1|2] ... Handle different input types: (0 for text,
1 for confusion network, 2 for lattices,
default is 0)
";
exit 1;
}
############################################################
############################################################
############################################################
# main
# we run moses to check validity of moses.ini and to obtain all the feature
# names
my $featlist = get_featlist_from_moses($___CONFIG);
$featlist = insert_ranges_to_featlist($featlist, $___RANGES);
# Mark which features are disabled:
if (defined $___ACTIVATE_FEATURES) {
my %enabled = map { ($_, 1) } split /[, ]+/, $___ACTIVATE_FEATURES;
my %cnt;
for(my $i=0; $i<scalar(@{$featlist->{"names"}}); $i++) {
my $name = $featlist->{"names"}->[$i];
$cnt{$name} = 0 if !defined $cnt{$name};
$featlist->{"enabled"}->[$i] = $enabled{$name."_".$cnt{$name}};
$cnt{$name}++;
}
} else {
# all enabled
for(my $i=0; $i<scalar(@{$featlist->{"names"}}); $i++) {
$featlist->{"enabled"}->[$i] = 1;
}
}
print STDERR "MERT starting values and ranges for random generation:\n";
for(my $i=0; $i<scalar(@{$featlist->{"names"}}); $i++) {
my $name = $featlist->{"names"}->[$i];
my $val = $featlist->{"values"}->[$i];
my $min = $featlist->{"mins"}->[$i];
my $max = $featlist->{"maxs"}->[$i];
my $enabled = $featlist->{"enabled"}->[$i];
printf STDERR " %5s = %7.3f", $name, $val;
if ($enabled) {
printf STDERR " (%5.2f .. %5.2f)\n", $min, $max;
} else {
print STDERR " --- inactive, not optimized ---\n";
}
}
sub get_featlist_from_moses {
# run moses with the given config file and return the list of features and
# their initial values
my $configfn = shift;
my $featlistfn = "./features.list";
if (-e $featlistfn) {
print STDERR "Using cached features list: $featlistfn\n";
} else {
print STDERR "Asking moses for feature names and values from $___CONFIG\n";
my $cmd = "$___DECODER $___DECODER_FLAGS -config $configfn -inputtype $___INPUTTYPE -show-weights > $featlistfn";
print STDERR "$cmd\n"; #DEBUG
safesystem($cmd) or die "Failed to run moses with the config $configfn";
}
# read feature list
my @names = ();
my @startvalues = ();
open(INI,$featlistfn) or die "Can't read $featlistfn";
my $nr = 0;
my @errs = ();
while (<INI>) {
$nr++;
chomp;
/^(.+) (\S+) (\S+)$/ || die("invalid feature: $_");
my ($longname, $feature, $value) = ($1,$2,$3);
next if $value eq "sparse";
push @errs, "$featlistfn:$nr:Bad initial value of $feature: $value\n"
if $value !~ /^[+-]?[0-9.e]+$/;
push @errs, "$featlistfn:$nr:Unknown feature '$feature', please add it to \@ABBR_FULL_MAP\n"
if !defined $ABBR2FULL{$feature};
push @names, $feature;
push @startvalues, $value;
}
close INI;
if (scalar @errs) {
print STDERR join("", @errs);
exit 1;
}
return {"names"=>\@names, "values"=>\@startvalues};
}
sub insert_ranges_to_featlist {
my $featlist = shift;
my $ranges = shift;
$ranges = [] if !defined $ranges;
# first collect the ranges from options
my $niceranges;
foreach my $range (@$ranges) {
my $name = undef;
foreach my $namedpair (split /,/, $range) {
if ($namedpair =~ /^(.*?):/) {
$name = $1;
$namedpair =~ s/^.*?://;
die "Unrecognized name '$name' in --range=$range"
if !defined $ABBR2FULL{$name};
}
my ($min, $max) = split /\.\./, $namedpair;
die "Bad min '$min' in --range=$range" if $min !~ /^-?[0-9.]+$/;
die "Bad max '$max' in --range=$range" if $min !~ /^-?[0-9.]+$/;
die "No name given in --range=$range" if !defined $name;
push @{$niceranges->{$name}}, [$min, $max];
}
}
# now populate featlist
my $seen = undef;
for(my $i=0; $i<scalar(@{$featlist->{"names"}}); $i++) {
my $name = $featlist->{"names"}->[$i];
$seen->{$name} ++;
my $min = 0.0;
my $max = 1.0;
if (defined $niceranges->{$name}) {
my $minmax = shift @{$niceranges->{$name}};
($min, $max) = @$minmax if defined $minmax;
}
$featlist->{"mins"}->[$i] = $min;
$featlist->{"maxs"}->[$i] = $max;
}
return $featlist;
}
sub safesystem {
print STDERR "Executing: @_\n";
system(@_);
if ($? == -1) {
print STDERR "Failed to execute: @_\n $!\n";
exit(1);
}
elsif ($? & 127) {
printf STDERR "Execution of: @_\n died with signal %d, %s coredump\n",
($? & 127), ($? & 128) ? 'with' : 'without';
exit(1);
}
else {
my $exitcode = $? >> 8;
print STDERR "Exit code: $exitcode\n" if $exitcode;
return ! $exitcode;
}
}

832
contrib/mert-sge-nosync/training/sge-nosync/zipextract-decoder-result.pl Executable file
View File

@ -0,0 +1,832 @@
#!/usr/bin/perl -w
# $Id$
# Usage:
# mert-moses.pl <foreign> <english> <decoder-executable> <decoder-config>
# For other options see below or run 'mert-moses.pl --help'
# Notes:
# <foreign> and <english> should be raw text files, one sentence per line
# <english> can be a prefix, in which case the files are <english>0, <english>1, etc. are used
# Excerpts from revision history
# Sept 2011 multi-threaded mert (Barry Haddow)
# 3 Aug 2011 Added random directions, historic best, pairwise ranked (PK)
# Jul 2011 simplifications (Ondrej Bojar)
# -- rely on moses' -show-weights instead of parsing moses.ini
# ... so moses is also run once *before* mert starts, checking
# the model to some extent
# -- got rid of the 'triples' mess;
# use --range to supply bounds for random starting values:
# --range tm:-3..3 --range lm:-3..3
# 5 Aug 2009 Handling with different reference length policies (shortest, average, closest) for BLEU
# and case-sensistive/insensitive evaluation (Nicola Bertoldi)
# 5 Jun 2008 Forked previous version to support new mert implementation.
# 13 Feb 2007 Better handling of default values for lambda, now works with multiple
# models and lexicalized reordering
# 11 Oct 2006 Handle different input types through parameter --inputype=[0|1]
# (0 for text, 1 for confusion network, default is 0) (Nicola Bertoldi)
# 10 Oct 2006 Allow skip of filtering of phrase tables (--no-filter-phrase-table)
# useful if binary phrase tables are used (Nicola Bertoldi)
# 28 Aug 2006 Use either closest or average or shortest (default) reference
# length as effective reference length
# Use either normalization or not (default) of texts (Nicola Bertoldi)
# 31 Jul 2006 move gzip run*.out to avoid failure wit restartings
# adding default paths
# 29 Jul 2006 run-filter, score-nbest and mert run on the queue (Nicola; Ondrej had to type it in again)
# 28 Jul 2006 attempt at foolproof usage, strong checking of input validity, merged the parallel and nonparallel version (Ondrej Bojar)
# 27 Jul 2006 adding the safesystem() function to handle with process failure
# 22 Jul 2006 fixed a bug about handling relative path of configuration file (Nicola Bertoldi)
# 21 Jul 2006 adapted for Moses-in-parallel (Nicola Bertoldi)
# 18 Jul 2006 adapted for Moses and cleaned up (PK)
# 21 Jan 2005 unified various versions, thorough cleanup (DWC)
# now indexing accumulated n-best list solely by feature vectors
# 14 Dec 2004 reimplemented find_threshold_points in C (NMD)
# 25 Oct 2004 Use either average or shortest (default) reference
# length as effective reference length (DWC)
# 13 Oct 2004 Use alternative decoders (DWC)
# Original version by Philipp Koehn
use strict;
use FindBin qw($Bin);
use File::Basename;
use File::Path;
use File::Spec;
use Cwd;
my $SCRIPTS_ROOTDIR = $Bin;
$SCRIPTS_ROOTDIR =~ s/\/training$//;
$SCRIPTS_ROOTDIR = $ENV{"SCRIPTS_ROOTDIR"} if defined($ENV{"SCRIPTS_ROOTDIR"});
## We preserve this bit of comments to keep the traditional weight ranges.
# "w" => [ [ 0.0, -1.0, 1.0 ] ], # word penalty
# "d" => [ [ 1.0, 0.0, 2.0 ] ], # lexicalized reordering model
# "lm" => [ [ 1.0, 0.0, 2.0 ] ], # language model
# "g" => [ [ 1.0, 0.0, 2.0 ], # generation model
# [ 1.0, 0.0, 2.0 ] ],
# "tm" => [ [ 0.3, 0.0, 0.5 ], # translation model
# [ 0.2, 0.0, 0.5 ],
# [ 0.3, 0.0, 0.5 ],
# [ 0.2, 0.0, 0.5 ],
# [ 0.0,-1.0, 1.0 ] ], # ... last weight is phrase penalty
# "lex"=> [ [ 0.1, 0.0, 0.2 ] ], # global lexical model
# "I" => [ [ 0.0,-1.0, 1.0 ] ], # input lattice scores
# moses.ini file uses FULL names for lambdas, while this training script
# internally (and on the command line) uses ABBR names.
my @ABBR_FULL_MAP = qw(d=weight-d lm=weight-l tm=weight-t w=weight-w
g=weight-generation lex=weight-lex I=weight-i);
my %ABBR2FULL = map {split/=/,$_,2} @ABBR_FULL_MAP;
my %FULL2ABBR = map {my ($a, $b) = split/=/,$_,2; ($b, $a);} @ABBR_FULL_MAP;
my $minimum_required_change_in_weights = 0.00001;
# stop if no lambda changes more than this
my $verbose = 0;
my $usage = 0; # request for --help
# We assume that if you don't specify working directory,
# we set the default is set to `pwd`/mert-work
#@@# my $___WORKING_DIR = File::Spec->catfile(Cwd::getcwd(), "mert-work");
my $___WORKING_DIR = undef;
my $___DEV_F = undef; # required, input text to decode
my $___DEV_E = undef; # required, basename of files with references
my $___DECODER = undef; # required, pathname to the decoder executable
my $___CONFIG = undef; # required, pathname to startup ini file
my $___N_BEST_LIST_SIZE = 100;
my $___LATTICE_SAMPLES = 0;
my $submithost = "";
my $queue_flags = "-hard"; # extra parameters for parallelizer
# the -l ws0ssmt was relevant only to JHU 2006 workshop
my $___JOBS = undef; # if parallel, number of jobs to use (undef or 0 -> serial)
my $___DECODER_FLAGS = ""; # additional parametrs to pass to the decoder
my $continue = 0; # should we try to continue from the last saved step?
my $skip_decoder = 0; # and should we skip the first decoder run (assuming we got interrupted during mert)
my $___FILTER_PHRASE_TABLE = 1; # filter phrase table
my $___PREDICTABLE_SEEDS = 0;
my $___START_WITH_HISTORIC_BESTS = 0; # use best settings from all previous iterations as starting points [Foster&Kuhn,2009]
my $___RANDOM_DIRECTIONS = 0; # search in random directions only
my $___NUM_RANDOM_DIRECTIONS = 0; # number of random directions, also works with default optimizer [Cer&al.,2008]
my $___PAIRWISE_RANKED_OPTIMIZER = 0; # use Hopkins&May[2011]
my $___PRO_STARTING_POINT = 0; # get a starting point from pairwise ranked optimizer
my $___RANDOM_RESTARTS = 20;
my $___HISTORIC_INTERPOLATION = 0; # interpolate optimize weights with previous iteration's weights [Hopkins&May,2011,5.4.3]
my $__THREADS = 0;
my $run = 0;
# Parameter for effective reference length when computing BLEU score
# Default is to use shortest reference
# Use "--shortest" to use shortest reference length
# Use "--average" to use average reference length
# Use "--closest" to use closest reference length
# Only one between --shortest, --average and --closest can be set
# If more than one choice the defualt (--shortest) is used
my $___SHORTEST = 0;
my $___AVERAGE = 0;
my $___CLOSEST = 0;
# Use "--nocase" to compute case-insensitive scores
my $___NOCASE = 0;
# Use "--nonorm" to non normalize translation before computing scores
my $___NONORM = 0;
# set 0 if input type is text, set 1 if input type is confusion network
my $___INPUTTYPE = 0;
my $mertdir = undef; # path to new mert directory
my $mertargs = undef; # args to pass through to mert & extractor
my $mertmertargs = undef; # args to pass through to mert only
my $extractorargs = undef; # args to pass through to extractor only
my $filtercmd = undef; # path to filter-model-given-input.pl
my $filterfile = undef;
my $qsubwrapper = undef;
my $qsubwrapper_exit = undef;
my $moses_parallel_cmd = undef;
my $old_sge = 0; # assume sge<6.0
my $___CONFIG_ORIG = undef; # pathname to startup ini file before filtering
my $___ACTIVATE_FEATURES = undef; # comma-separated (or blank-separated) list of features to work on
# if undef work on all features
# (others are fixed to the starting values)
my $___RANGES = undef;
my $prev_aggregate_nbl_size = -1; # number of previous step to consider when loading data (default =-1)
# -1 means all previous, i.e. from iteration 1
# 0 means no previous data, i.e. from actual iteration
# 1 means 1 previous data , i.e. from the actual iteration and from the previous one
# and so on
my $maximum_iterations = 25;
#####################
my $processfeatlistcmd = undef;
my $processfeatlistargs = undef;
my $createconfigcmd = undef;
my $createconfigargs = undef;
my $decoderargs = undef;
#####################
use Getopt::Long;
GetOptions(
"working-dir=s" => \$___WORKING_DIR,
"input=s" => \$___DEV_F,
"inputtype=i" => \$___INPUTTYPE,
"refs=s" => \$___DEV_E,
"decoder=s" => \$___DECODER,
"config=s" => \$___CONFIG,
"nbest=i" => \$___N_BEST_LIST_SIZE,
"lattice-samples=i" => \$___LATTICE_SAMPLES,
"submithost=s" => \$submithost,
"queue-flags=s" => \$queue_flags,
"jobs=i" => \$___JOBS,
"decoder-flags=s" => \$___DECODER_FLAGS,
"continue" => \$continue,
"skip-decoder" => \$skip_decoder,
"shortest" => \$___SHORTEST,
"average" => \$___AVERAGE,
"closest" => \$___CLOSEST,
"nocase" => \$___NOCASE,
"nonorm" => \$___NONORM,
"help" => \$usage,
"verbose" => \$verbose,
"mertdir=s" => \$mertdir,
"mertargs=s" => \$mertargs,
"extractorargs=s" => \$extractorargs,
"mertmertargs=s" => \$mertmertargs,
"rootdir=s" => \$SCRIPTS_ROOTDIR,
"filtercmd=s" => \$filtercmd, # allow to override the default location
"filterfile=s" => \$filterfile, # input to filtering script (useful for lattices/confnets)
"qsubwrapper=s" => \$qsubwrapper, # allow to override the default location
"mosesparallelcmd=s" => \$moses_parallel_cmd, # allow to override the default location
"old-sge" => \$old_sge, #passed to moses-parallel
"filter-phrase-table!" => \$___FILTER_PHRASE_TABLE, # (dis)allow of phrase tables
"predictable-seeds" => \$___PREDICTABLE_SEEDS, # make random restarts deterministic
"historic-bests" => \$___START_WITH_HISTORIC_BESTS, # use best settings from all previous iterations as starting points
"random-directions" => \$___RANDOM_DIRECTIONS, # search only in random directions
"run=i" => \$run,
"number-of-random-directions=i" => \$___NUM_RANDOM_DIRECTIONS, # number of random directions
"random-restarts=i" => \$___RANDOM_RESTARTS, # number of random restarts
"activate-features=s" => \$___ACTIVATE_FEATURES, #comma-separated (or blank-separated) list of features to work on (others are fixed to the starting values)
"range=s@" => \$___RANGES,
"prev-aggregate-nbestlist=i" => \$prev_aggregate_nbl_size, #number of previous step to consider when loading data (default =-1, i.e. all previous)
"maximum-iterations=i" => \$maximum_iterations,
"pairwise-ranked" => \$___PAIRWISE_RANKED_OPTIMIZER,
"pro-starting-point" => \$___PRO_STARTING_POINT,
"historic-interpolation=f" => \$___HISTORIC_INTERPOLATION,
"threads=i" => \$__THREADS
) or exit(1);
# the 4 required parameters can be supplied on the command line directly
# or using the --options
if (scalar @ARGV == 2) {
# required parameters: input_file references_basename decoder_executable
# $___DEV_F = shift;
$___DEV_E = shift;
# $___DECODER = shift;
$___CONFIG = shift;
}
# if ($usage || !defined $___DEV_F || !defined $___DEV_E || !defined $___DECODER || !defined $___CONFIG) {
if ($usage || !defined $___CONFIG || !defined $___DEV_E ){
print STDERR "usage: $0 reference decoder.ini
Options:
--working-dir=mert-dir ... where all the files are created
--nbest=100 ... how big nbestlist to generate
--lattice-samples ... how many lattice samples (Chatterjee & Cancedda, emnlp 2010)
--jobs=N ... set this to anything to run moses in parallel
--mosesparallelcmd=STR ... use a different script instead of moses-parallel
--queue-flags=STRING ... anything you with to pass to qsub, eg.
'-l ws06osssmt=true'. The default is: '-hard'
To reset the parameters, please use
--queue-flags=' '
(i.e. a space between the quotes).
--decoder-flags=STRING ... extra parameters for the decoder
--continue ... continue from the last successful iteration
--skip-decoder ... skip the decoder run for the first time,
assuming that we got interrupted during
optimization
--shortest --average --closest
... Use shortest/average/closest reference length
as effective reference length (mutually exclusive)
--nocase ... Do not preserve case information; i.e.
case-insensitive evaluation (default is false).
--nonorm ... Do not use text normalization (flag is not active,
i.e. text is NOT normalized)
--filtercmd=STRING ... path to filter-model-given-input.pl
--filterfile=STRING ... path to alternative to input-text for filtering
model. useful for lattice decoding
--rootdir=STRING ... where do helpers reside (if not given explicitly)
--mertdir=STRING ... path to new mert implementation
--mertargs=STRING ... extra args for both extractor and mert
--extractorargs=STRING ... extra args for extractor only
--mertmertargs=STRING ... extra args for mert only
--scorenbestcmd=STRING ... path to score-nbest.py
--old-sge ... passed to parallelizers, assume Grid Engine < 6.0
--inputtype=[0|1|2] ... Handle different input types: (0 for text,
1 for confusion network, 2 for lattices,
default is 0)
--no-filter-phrase-table ... disallow filtering of phrase tables
(useful if binary phrase tables are available)
--random-restarts=INT ... number of random restarts (default: 20)
--predictable-seeds ... provide predictable seeds to mert so that random
restarts are the same on every run
--range=tm:0..1,-1..1 ... specify min and max value for some features
--range can be repeated as needed.
The order of the various --range specifications
is important only within a feature name.
E.g.:
--range=tm:0..1,-1..1 --range=tm:0..2
is identical to:
--range=tm:0..1,-1..1,0..2
but not to:
--range=tm:0..2 --range=tm:0..1,-1..1
--activate-features=STRING ... comma-separated list of features to optimize,
others are fixed to the starting values
default: optimize all features
example: tm_0,tm_4,d_0
--prev-aggregate-nbestlist=INT ... number of previous step to consider when
loading data (default = $prev_aggregate_nbl_size)
-1 means all previous, i.e. from iteration 1
0 means no previous data, i.e. only the
current iteration
N means this and N previous iterations
--maximum-iterations=ITERS ... Maximum number of iterations. Default: $maximum_iterations
--random-directions ... search only in random directions
--number-of-random-directions=int ... number of random directions
(also works with regular optimizer, default: 0)
--pairwise-ranked ... Use PRO for optimisation (Hopkins and May, emnlp 2011)
--pro-starting-point ... Use PRO to get a starting point for MERT
--threads=NUMBER ... Use multi-threaded mert (must be compiled in).
--historic-interpolation ... Interpolate optimized weights with prior iterations' weight
(parameter sets factor [0;1] given to current weights)
";
exit 1;
}
# Check validity of input parameters and set defaults if needed
print STDERR "Using SCRIPTS_ROOTDIR: $SCRIPTS_ROOTDIR\n";
# path of script for filtering phrase tables and running the decoder
$filtercmd="$SCRIPTS_ROOTDIR/training/filter-model-given-input.pl" if !defined $filtercmd;
if ( ! -x $filtercmd && ! $___FILTER_PHRASE_TABLE) {
print STDERR "Filtering command not found: $filtercmd.\n";
print STDERR "Use --filtercmd=PATH to specify a valid one or --no-filter-phrase-table\n";
exit 1;
}
# $qsubwrapper="$SCRIPTS_ROOTDIR/generic/qsub-wrapper.pl" if !defined $qsubwrapper;
$qsubwrapper = "$SCRIPTS_ROOTDIR/generic/qsub-wrapper-sge-nosync.pl" if !defined $qsubwrapper;
$qsubwrapper_exit = "$SCRIPTS_ROOTDIR/generic/qsub-wrapper-exit-sge-nosync.pl" if !defined $qsubwrapper_exit;
# $moses_parallel_cmd = "$SCRIPTS_ROOTDIR/generic/moses-parallel.pl"
# if !defined $moses_parallel_cmd;
$moses_parallel_cmd = "$SCRIPTS_ROOTDIR/generic/moses-parallel-sge-nosync.pl"
if !defined $moses_parallel_cmd;
if (!defined $mertdir) {
$mertdir = "$SCRIPTS_ROOTDIR/../mert";
print STDERR "Assuming --mertdir=$mertdir\n";
}
my $mert_extract_cmd = "$mertdir/extractor";
my $mert_mert_cmd = "$mertdir/mert";
my $mert_pro_cmd = "$mertdir/pro";
die "Not executable: $mert_extract_cmd" if ! -x $mert_extract_cmd;
die "Not executable: $mert_mert_cmd" if ! -x $mert_mert_cmd;
die "Not executable: $mert_pro_cmd" if ! -x $mert_pro_cmd;
my $pro_optimizer = "$mertdir/megam_i686.opt"; # or set to your installation
if (($___PAIRWISE_RANKED_OPTIMIZER || $___PRO_STARTING_POINT) && ! -x $pro_optimizer) {
print "did not find $pro_optimizer, installing it in $mertdir\n";
`cd $mertdir; wget http://www.cs.utah.edu/~hal/megam/megam_i686.opt.gz;`;
`gunzip $pro_optimizer.gz`;
`chmod +x $pro_optimizer`;
die("ERROR: Installation of megam_i686.opt failed! Install by hand from http://www.cs.utah.edu/~hal/megam/") unless -x $pro_optimizer;
}
$mertargs = "" if !defined $mertargs;
my $scconfig = undef;
if ($mertargs =~ /\-\-scconfig\s+(.+?)(\s|$)/){
$scconfig=$1;
$scconfig =~ s/\,/ /g;
$mertargs =~ s/\-\-scconfig\s+(.+?)(\s|$)//;
}
# handling reference lengh strategy
if (($___CLOSEST + $___AVERAGE + $___SHORTEST) > 1){
die "You can specify just ONE reference length strategy (closest or shortest or average) not both\n";
}
if ($___SHORTEST){
$scconfig .= " reflen:shortest";
}elsif ($___AVERAGE){
$scconfig .= " reflen:average";
}elsif ($___CLOSEST){
$scconfig .= " reflen:closest";
}
# handling case-insensitive flag
if ($___NOCASE) {
$scconfig .= " case:false";
}else{
$scconfig .= " case:true";
}
$scconfig =~ s/^\s+//;
$scconfig =~ s/\s+$//;
$scconfig =~ s/\s+/,/g;
$scconfig = "--scconfig $scconfig" if ($scconfig);
my $mert_extract_args=$mertargs;
$mert_extract_args .=" $scconfig";
$mert_extract_args .=" $extractorargs";
$mertmertargs = "" if !defined $mertmertargs;
my $mert_mert_args="$mertargs $mertmertargs";
$mert_mert_args =~ s/\-+(binary|b)\b//;
$mert_mert_args .=" $scconfig";
if ($___ACTIVATE_FEATURES){ $mert_mert_args .=" -o \"$___ACTIVATE_FEATURES\""; }
my ($just_cmd_filtercmd,$x) = split(/ /,$filtercmd);
die "Not executable: $just_cmd_filtercmd" if ! -x $just_cmd_filtercmd;
die "Not executable: $moses_parallel_cmd" if defined $___JOBS && ! -x $moses_parallel_cmd;
die "Not executable: $qsubwrapper" if defined $___JOBS && ! -x $qsubwrapper;
#@@# die "Not executable: $___DECODER" if ! -x $___DECODER;
#@@# my $input_abs = ensure_full_path($___DEV_F);
#@@# die "File not found: $___DEV_F (interpreted as $input_abs)."
#@@# if ! -e $input_abs;
#@@# $___DEV_F = $input_abs;
# Option to pass to qsubwrapper and moses-parallel
my $pass_old_sge = $old_sge ? "-old-sge" : "";
#@@# my $decoder_abs = ensure_full_path($___DECODER);
#@@# die "File not executable: $___DECODER (interpreted as $decoder_abs)."
#@@# if ! -x $decoder_abs;
#@@# $___DECODER = $decoder_abs;
my $ref_abs = ensure_full_path($___DEV_E);
# check if English dev set (reference translations) exist and store a list of all references
my @references;
if (-e $ref_abs) {
push @references, $ref_abs;
}
else {
# if multiple file, get a full list of the files
my $part = 0;
if (! -e $ref_abs."0" && -e $ref_abs.".ref0") {
$ref_abs .= ".ref";
}
while (-e $ref_abs.$part) {
push @references, $ref_abs.$part;
$part++;
}
die("Reference translations not found: $___DEV_E (interpreted as $ref_abs)") unless $part;
}
my $config_abs = ensure_full_path($___CONFIG);
die "File not found: $___CONFIG (interpreted as $config_abs)."
if ! -e $config_abs;
$___CONFIG = $config_abs;
# moses should use our config
if ($___DECODER_FLAGS =~ /(^|\s)-(config|f) /
|| $___DECODER_FLAGS =~ /(^|\s)-(ttable-file|t) /
|| $___DECODER_FLAGS =~ /(^|\s)-(distortion-file) /
|| $___DECODER_FLAGS =~ /(^|\s)-(generation-file) /
|| $___DECODER_FLAGS =~ /(^|\s)-(lmodel-file) /
|| $___DECODER_FLAGS =~ /(^|\s)-(global-lexical-file) /
) {
die "It is forbidden to supply any of -config, -ttable-file, -distortion-file, -generation-file or -lmodel-file in the --decoder-flags.\nPlease use only the --config option to give the config file that lists all the supplementary files.";
}
# as weights are normalized in the next steps (by cmert)
# normalize initial LAMBDAs, too
my $need_to_normalize = 1;
#store current directory and create the working directory (if needed)
my $cwd = `pawd 2>/dev/null`;
if(!$cwd){$cwd = `pwd`;}
chomp($cwd);
$___WORKING_DIR = $cwd if (!defined $___WORKING_DIR);
chomp $___WORKING_DIR;
print STDERR "working dir is $___WORKING_DIR\n";
#@@# mkpath($___WORKING_DIR);
{
# open local scope
#chdir to the working directory
chdir($___WORKING_DIR) or die "Can't chdir to $___WORKING_DIR";
# fixed file names
my $mert_outfile = "mert.out";
my $mert_logfile = "mert.log";
my $weights_in_file = "init.opt";
my $weights_out_file = "weights.txt";
# set start run
my $start_run = 1;
my $bestpoint = undef;
my $devbleu = undef;
my $sparse_weights_file = undef;
my $jobid = -1;
my $prev_feature_file = undef;
my $prev_score_file = undef;
my $prev_init_file = undef;
#########################
# set jobid to trace different jobs
my $prevjid = undef;
#### my $run=$start_run-1;
my $oldallsorted = undef;
my $allsorted = undef;
my $nbest_file=undef;
my $lsamp_file=undef; #Lattice samples
my $orig_nbest_file=undef; # replaced if lattice sampling
my $cmd=undef;
$nbest_file="run$run.best$___N_BEST_LIST_SIZE.out";
safesystem("gzip -f $nbest_file") or die "Failed to gzip run*out";
$nbest_file = $nbest_file.".gz";
# extract score statistics and features from the nbest lists
print STDERR "Scoring the nbestlist.\n";
my $base_feature_file = "features.dat";
my $base_score_file = "scores.dat";
my $feature_file = "run$run.${base_feature_file}";
my $score_file = "run$run.${base_score_file}";
$cmd = "$mert_extract_cmd $mert_extract_args --scfile $score_file --ffile $feature_file -r ".join(",", @references)." -n $nbest_file";
$cmd = create_extractor_script($cmd, $___WORKING_DIR);
&submit_or_exec($cmd,"extract.out","extract.err","extract.id");
} # end of local scope
sub get_weights_from_mert {
my ($outfile,$logfile,$weight_count,$sparse_weights) = @_;
my ($bestpoint,$devbleu);
if ($___PAIRWISE_RANKED_OPTIMIZER || ($___PRO_STARTING_POINT && $logfile =~ /pro/)) {
open(IN,$outfile) or die "Can't open $outfile";
my (@WEIGHT,$sum);
for(my $i=0;$i<$weight_count;$i++) { push @WEIGHT, 0; }
while(<IN>) {
# regular features
if (/^F(\d+) ([\-\.\de]+)/) {
$WEIGHT[$1] = $2;
$sum += abs($2);
}
# sparse features
elsif(/^(.+_.+) ([\-\.\de]+)/) {
$$sparse_weights{$1} = $2;
}
}
$devbleu = "unknown";
foreach (@WEIGHT) { $_ /= $sum; }
foreach (keys %{$sparse_weights}) { $$sparse_weights{$_} /= $sum; }
$bestpoint = join(" ",@WEIGHT);
close IN;
}
else {
open(IN,$logfile) or die "Can't open $logfile";
while (<IN>) {
if (/Best point:\s*([\s\d\.\-e]+?)\s*=> ([\-\d\.]+)/) {
$bestpoint = $1;
$devbleu = $2;
last;
}
}
close IN;
}
return ($bestpoint,$devbleu);
}
sub sanity_check_order_of_lambdas {
my $featlist = shift;
my $filename_or_stream = shift;
my @expected_lambdas = @{$featlist->{"names"}};
my @got = get_order_of_scores_from_nbestlist($filename_or_stream);
die "Mismatched lambdas. Decoder returned @got, we expected @expected_lambdas"
if "@got" ne "@expected_lambdas";
}
sub get_featlist_from_moses {
# run moses with the given config file and return the list of features and
# their initial values
my $configfn = shift;
my $featlistfn = "./features.list";
if (-e $featlistfn) {
print STDERR "Using cached features list: $featlistfn\n";
} else {
print STDERR "Asking moses for feature names and values from $___CONFIG\n";
my $cmd = "$___DECODER $___DECODER_FLAGS -config $configfn -inputtype $___INPUTTYPE -show-weights > $featlistfn";
safesystem($cmd) or die "Failed to run moses with the config $configfn";
}
# read feature list
my @names = ();
my @startvalues = ();
open(INI,$featlistfn) or die "Can't read $featlistfn";
my $nr = 0;
my @errs = ();
while (<INI>) {
$nr++;
chomp;
/^(.+) (\S+) (\S+)$/ || die("invalid feature: $_");
my ($longname, $feature, $value) = ($1,$2,$3);
next if $value eq "sparse";
push @errs, "$featlistfn:$nr:Bad initial value of $feature: $value\n"
if $value !~ /^[+-]?[0-9.e]+$/;
push @errs, "$featlistfn:$nr:Unknown feature '$feature', please add it to \@ABBR_FULL_MAP\n"
if !defined $ABBR2FULL{$feature};
push @names, $feature;
push @startvalues, $value;
}
close INI;
if (scalar @errs) {
print STDERR join("", @errs);
exit 1;
}
return {"names"=>\@names, "values"=>\@startvalues};
}
sub get_order_of_scores_from_nbestlist {
# read the first line and interpret the ||| label: num num num label2: num ||| column in nbestlist
# return the score labels in order
my $fname_or_source = shift;
# print STDERR "Peeking at the beginning of nbestlist to get order of scores: $fname_or_source\n";
open IN, $fname_or_source or die "Failed to get order of scores from nbestlist '$fname_or_source'";
my $line = <IN>;
close IN;
die "Line empty in nbestlist '$fname_or_source'" if !defined $line;
my ($sent, $hypo, $scores, $total) = split /\|\|\|/, $line;
$scores =~ s/^\s*|\s*$//g;
die "No scores in line: $line" if $scores eq "";
my @order = ();
my $label = undef;
my $sparse = 0; # we ignore sparse features here
foreach my $tok (split /\s+/, $scores) {
if ($tok =~ /.+_.+:/) {
$sparse = 1;
} elsif ($tok =~ /^([a-z][0-9a-z]*):/i) {
$label = $1;
} elsif ($tok =~ /^-?[-0-9.e]+$/) {
if (!$sparse) {
# a score found, remember it
die "Found a score but no label before it! Bad nbestlist '$fname_or_source'!"
if !defined $label;
push @order, $label;
}
$sparse = 0;
} else {
die "Not a label, not a score '$tok'. Failed to parse the scores string: '$scores' of nbestlist '$fname_or_source'";
}
}
print STDERR "The decoder returns the scores in this order: @order\n";
return @order;
}
sub safesystem {
print STDERR "Executing: @_\n";
system(@_);
if ($? == -1) {
print STDERR "Failed to execute: @_\n $!\n";
exit(1);
}
elsif ($? & 127) {
printf STDERR "Execution of: @_\n died with signal %d, %s coredump\n",
($? & 127), ($? & 128) ? 'with' : 'without';
exit(1);
}
else {
my $exitcode = $? >> 8;
print STDERR "Exit code: $exitcode\n" if $exitcode;
return ! $exitcode;
}
}
sub ensure_full_path {
my $PATH = shift;
$PATH =~ s/\/nfsmnt//;
return $PATH if $PATH =~ /^\//;
my $dir = `pawd 2>/dev/null`;
if(!$dir){$dir = `pwd`;}
chomp($dir);
$PATH = $dir."/".$PATH;
$PATH =~ s/[\r\n]//g;
$PATH =~ s/\/\.\//\//g;
$PATH =~ s/\/+/\//g;
my $sanity = 0;
while($PATH =~ /\/\.\.\// && $sanity++<10) {
$PATH =~ s/\/+/\//g;
$PATH =~ s/\/[^\/]+\/\.\.\//\//g;
}
$PATH =~ s/\/[^\/]+\/\.\.$//;
$PATH =~ s/\/+$//;
$PATH =~ s/\/nfsmnt//;
return $PATH;
}
sub submit_or_exec {
my $argvlen = @_;
my $cmd = undef;
my $stdout = undef;
my $stderr = undef;
my $jidfile = undef;
my $prevjid = undef;
# if supply 3 arguments, exec without submit
# if supply 4 arguments, then submit new job
# if supply 5 arguments, wait for the previous job to finish
if ($argvlen == 3){
($cmd,$stdout,$stderr) = @_;
} elsif ($argvlen == 4){
($cmd,$stdout,$stderr,$jidfile) = @_;
} elsif ($argvlen == 5){
($cmd,$stdout,$stderr,$jidfile,$prevjid) = @_;
}
print STDERR "exec: $cmd\n";
if (defined $___JOBS && $___JOBS > 0 && $argvlen==5) {
safesystem("$qsubwrapper $pass_old_sge -command='$cmd' -queue-parameters=\"$queue_flags\" -stdout=$stdout -stderr=$stderr -jidfile=$jidfile -prevjid=$prevjid" )
or die "ERROR: Failed to submit '$cmd' (via $qsubwrapper)";
}
elsif (defined $___JOBS && $___JOBS > 0 && $argvlen==4) {
safesystem("$qsubwrapper $pass_old_sge -command='$cmd' -queue-parameters=\"$queue_flags\" -stdout=$stdout -stderr=$stderr -jidfile=$jidfile" )
or die "ERROR: Failed to submit '$cmd' (via $qsubwrapper)";
} else {
safesystem("$cmd > $stdout 2> $stderr") or die "ERROR: Failed to run '$cmd'.";
}
}
sub exit_submit {
my $argvlen = @_;
my $cmd = undef;
my $stdout = undef;
my $stderr = undef;
my $jidfile = undef;
my $pidfile = undef;
my $prevjid = undef;
my $prevjidarraysize = 0;
my @prevjidarray = ();
my $pid = undef;
my $qsubcmd="";
my $hj="";
# if supply 4 arguments, then submit new job
# if supply 5 arguments, wait for the previous job to finish
if ($argvlen == 2) {
($stdout,$stderr) = @_;
} elsif ($argvlen == 4){
($stdout,$stderr,$jidfile,$pidfile) = @_;
} elsif ($argvlen == 5){
($stdout,$stderr,$jidfile,$pidfile,$prevjid) = @_;
}
# parse prevjid ########################
$prevjid =~ s/^\s+|\s+$//g;
@prevjidarray = split(/\s+/,$prevjid);
$prevjidarraysize = scalar(@prevjidarray);
########################################
# print STDERR "exec: $stdout\n";
# read pid from file, and draft exit script ##################
chomp ($pid=`tail -n 1 $pidfile`);
open (OUT, ">exitjob$pid.sh");
my $scriptheader="\#\!/bin/bash\n\#\$ -S /bin/sh\n# Both lines are needed to invoke base\n#the above line is ignored by qsub, unless parameter \"-b yes\" is set!\n\n";
$scriptheader .="uname -a\n\n";
$scriptheader .="cd $___WORKING_DIR\n\n";
print OUT $scriptheader;
print OUT "if $qsubwrapper_exit -submithost=$submithost -stdout=$stdout -stderr=$stderr -jidfile=$jidfile -pidfile=$pidfile > exitjob$pid.out 2> exitjob$pid.err ; then
echo 'succeeded'
else
echo failed with exit status \$\?
die=1
fi
";
print OUT "\n\n";
close (OUT);
# setting permissions of the script
chmod(oct(755),"exitjob$pid.sh");
##############################################################
if (defined $___JOBS && $___JOBS > 0 && $argvlen==5) {
if (defined $prevjid && $prevjid!=-1 && $prevjidarraysize == 1){
$hj = "-hold_jid $prevjid";
} elsif (defined $prevjid && $prevjidarraysize > 1){
$hj = "-hold_jid " . join(" -hold_jid ", @prevjidarray);
}
$qsubcmd="qsub $queue_flags -V $hj exitjob$pid.sh > exitjob$pid.log 2>&1";
safesystem($qsubcmd) or die "ERROR: Failed to exit-submit $pid (via $qsubwrapper_exit)";
} elsif (defined $___JOBS && $___JOBS > 0 && $argvlen==4) {
$qsubcmd="qsub $queue_flags -V exitjob$pid.sh > exitjob$pid.log 2>&1";
safesystem($qsubcmd) or die "ERROR: Failed to exit-submit $pid (via $qsubwrapper_exit)";
} else {
safesystem("rm $stdout") or die "ERROR: Failed to remove '$stdout'.";
safesystem("rm $stderr") or die "ERROR: Failed to remove '$stderr'.";
}
}
sub create_extractor_script()
{
my ($cmd, $outdir) = @_;
my $script_path = File::Spec->catfile($outdir, "extractor.sh");
open my $out, '>', $script_path
or die "Couldn't open $script_path for writing: $!\n";
print $out "#!/bin/bash\n";
print $out "cd $outdir\n";
print $out "$cmd\n";
close($out);
`chmod +x $script_path`;
return $script_path;
}