mirror of
https://github.com/moses-smt/mosesdecoder.git
synced 2024-09-19 07:07:24 +03:00
3816 lines
129 KiB
Perl
Executable File
3816 lines
129 KiB
Perl
Executable File
#!/usr/bin/env perl
|
|
#
|
|
# This file is part of moses. Its use is licensed under the GNU Lesser General
|
|
# Public License version 2.1 or, at your option, any later version.
|
|
|
|
# Experiment Management System
|
|
# Documentation at http://www.statmt.org/moses/?n=FactoredTraining.EMS
|
|
|
|
use warnings;
|
|
use strict;
|
|
use Getopt::Long "GetOptions";
|
|
use FindBin qw($RealBin);
|
|
|
|
sub trim($)
|
|
{
|
|
my $string = shift;
|
|
$string =~ s/^\s+//;
|
|
$string =~ s/\s+$//;
|
|
return $string;
|
|
}
|
|
|
|
my $host = `hostname`; chop($host);
|
|
print STDERR "STARTING UP AS PROCESS $$ ON $host AT ".`date`;
|
|
|
|
my ($CONFIG_FILE,
|
|
$EXECUTE,
|
|
$NO_GRAPH,
|
|
$CONTINUE,
|
|
$FINAL_STEP,
|
|
$FINAL_OUT,
|
|
$VERBOSE,
|
|
$IGNORE_TIME,
|
|
$DELETE_CRASHED,
|
|
$DELETE_VERSION
|
|
);
|
|
|
|
my $SLEEP = 2;
|
|
my $META = "$RealBin/experiment.meta";
|
|
|
|
# check if it is run on a multi-core machine
|
|
# set number of maximal concurrently active processes
|
|
my ($MULTICORE,$MAX_ACTIVE) = (0,2);
|
|
&detect_if_multicore();
|
|
|
|
# check if running on a gridengine cluster
|
|
my $CLUSTER;
|
|
&detect_if_cluster();
|
|
|
|
# get command line options;
|
|
die("experiment.perl -config config-file [-exec] [-no-graph]")
|
|
unless &GetOptions('config=s' => \$CONFIG_FILE,
|
|
'continue=i' => \$CONTINUE,
|
|
'delete-crashed=i' => \$DELETE_CRASHED,
|
|
'delete-run=i' => \$DELETE_VERSION,
|
|
'delete-version=i' => \$DELETE_VERSION,
|
|
'ignore-time' => \$IGNORE_TIME,
|
|
'exec' => \$EXECUTE,
|
|
'cluster' => \$CLUSTER,
|
|
'multicore' => \$MULTICORE,
|
|
'final-step=s' => \$FINAL_STEP,
|
|
'final-out=s' => \$FINAL_OUT,
|
|
'meta=s' => \$META,
|
|
'verbose' => \$VERBOSE,
|
|
'sleep=i' => \$SLEEP,
|
|
'max-active=i' => \$MAX_ACTIVE,
|
|
'no-graph' => \$NO_GRAPH);
|
|
if (! -e "steps") { `mkdir -p steps`; }
|
|
|
|
die("error: could not find config file")
|
|
unless ($CONFIG_FILE && -e $CONFIG_FILE) ||
|
|
($CONTINUE && -e &steps_file("config.$CONTINUE",$CONTINUE)) ||
|
|
($DELETE_CRASHED && -e &steps_file("config.$DELETE_CRASHED",$DELETE_CRASHED)) ||
|
|
($DELETE_VERSION && -e &steps_file("config.$DELETE_VERSION",$DELETE_VERSION));
|
|
$CONFIG_FILE = &steps_file("config.$CONTINUE",$CONTINUE) if $CONTINUE && !$CONFIG_FILE;
|
|
$CONFIG_FILE = &steps_file("config.$DELETE_CRASHED",$DELETE_CRASHED) if $DELETE_CRASHED;
|
|
$CONFIG_FILE = &steps_file("config.$DELETE_VERSION",$DELETE_VERSION) if $DELETE_VERSION;
|
|
|
|
my (@MODULE,
|
|
%MODULE_TYPE,
|
|
%MODULE_STEP,
|
|
%STEP_IN,
|
|
%STEP_OUT,
|
|
%STEP_OUTNAME, # output file name for step result
|
|
%STEP_TMPNAME, # tmp directory to be used by step
|
|
%STEP_FINAL, # output is part of the final model, not an intermediate step
|
|
%STEP_PASS, # config parameters that have to be set, otherwise pass
|
|
%STEP_PASS_IF, # config parameters that have to be not set, otherwise pass
|
|
%STEP_IGNORE, # config parameters that have to be set, otherwise ignore
|
|
%STEP_IGNORE_IF, # config parameters that have to be not set, otherwise ignore
|
|
%QSUB_SCRIPT, # flag if script contains qsub's when run on cluster
|
|
%QSUB_STEP, # flag if step contains qsub's when run on cluster
|
|
%RERUN_ON_CHANGE, # config parameter whose change invalidates old runs
|
|
%ONLY_EXISTENCE_MATTERS, # re-use check only on existance, not value
|
|
%MULTIREF, # flag if step may be run on multiple sets (reference translations)
|
|
%TEMPLATE, # template if step follows a simple pattern
|
|
%TEMPLATE_IF, # part of template that is conditionally executed
|
|
%ONLY_FACTOR_0, # only run on a corpus that includes surface word
|
|
%PARALLELIZE, # flag, if step may be run through parallelizer
|
|
%ERROR, # patterns to check in stderr that indicate errors
|
|
%NOT_ERROR); # patterns that override general error indicating patterns
|
|
&read_meta();
|
|
|
|
print "LOAD CONFIG...\n";
|
|
my (@MODULE_LIST, # list of modules (included sets) used
|
|
%CONFIG); # all (expanded) parameter settings from configuration file
|
|
&read_config();
|
|
print "working directory is ".&check_and_get("GENERAL:working-dir")."\n";
|
|
chdir(&check_and_get("GENERAL:working-dir"));
|
|
|
|
my $VERSION = 0; # experiment number
|
|
$VERSION = $CONTINUE if $CONTINUE;
|
|
$VERSION = $DELETE_CRASHED if $DELETE_CRASHED;
|
|
$VERSION = $DELETE_VERSION if $DELETE_VERSION;
|
|
|
|
&compute_version_number() if $EXECUTE && !$CONTINUE && !$DELETE_CRASHED && !$DELETE_VERSION;
|
|
`mkdir -p steps/$VERSION` unless -d "steps/$VERSION";
|
|
|
|
&log_config() unless $DELETE_CRASHED || $DELETE_VERSION;
|
|
print "running experimental run number $VERSION\n";
|
|
|
|
print "\nESTABLISH WHICH STEPS NEED TO BE RUN\n";
|
|
my (%NEEDED, # mapping of input files to step numbers
|
|
%USES_INPUT, # mapping of step numbers to input files
|
|
@DO_STEP, # list of steps with fully specified name (LM:all:binarize)
|
|
%STEP_LOOKUP,# mapping from step name to step number
|
|
%PASS, # steps for which no action needs to be taken
|
|
%GIVEN); # mapping of given output files to fully specified name
|
|
&find_steps();
|
|
|
|
print "\nFIND DEPENDENCIES BETWEEN STEPS\n";
|
|
my @DEPENDENCY;
|
|
&find_dependencies();
|
|
|
|
if (defined($DELETE_CRASHED)) {
|
|
&delete_crashed($DELETE_CRASHED);
|
|
exit;
|
|
}
|
|
|
|
if (defined($DELETE_VERSION)) {
|
|
&delete_version($DELETE_VERSION);
|
|
exit;
|
|
}
|
|
|
|
print "\nCHECKING IF OLD STEPS ARE RE-USABLE\n";
|
|
my @RE_USE; # maps re-usable steps to older versions
|
|
my %RECURSIVE_RE_USE; # stores links from .INFO files that record prior re-use
|
|
&find_re_use();
|
|
|
|
print "\nDEFINE STEPS (run with -exec if everything ok)\n" unless $EXECUTE || $CONTINUE;
|
|
&define_step("all") unless $EXECUTE || $CONTINUE;
|
|
&init_agenda_graph();
|
|
&draw_agenda_graph();
|
|
|
|
print "\nEXECUTE STEPS\n" if $EXECUTE;
|
|
my (%DO,%DONE,%CRASHED); # tracks steps that are currently processed or done
|
|
&execute_steps() if $EXECUTE;
|
|
&draw_agenda_graph();
|
|
|
|
exit();
|
|
|
|
### SUB ROUTINES
|
|
# graph that depicts steps of the experiment, with depedencies
|
|
|
|
sub init_agenda_graph() {
|
|
my $dir = &check_and_get("GENERAL:working-dir");
|
|
|
|
my $graph_file = &steps_file("graph.$VERSION",$VERSION);
|
|
open(PS,">".$graph_file.".ps") or die "Cannot open: $!";
|
|
print PS "%!\n"
|
|
."/Helvetica findfont 36 scalefont setfont\n"
|
|
."72 72 moveto\n"
|
|
."(its all gone blank...) show\n"
|
|
."showpage\n";
|
|
close(PS);
|
|
|
|
`convert -alpha off $graph_file.ps $graph_file.png`;
|
|
|
|
if (!$NO_GRAPH && !fork) {
|
|
# use ghostview by default, it it is installed
|
|
if (`which gv 2> /dev/null`) {
|
|
`gv -watch $graph_file.ps`;
|
|
}
|
|
# ... otherwise use graphviz's display
|
|
else {
|
|
`display -update 10 $graph_file.png`;
|
|
}
|
|
#gotta exit the fork once the user has closed gv. Otherwise we'll have an extra instance of
|
|
#experiment.perl floating around running steps in parallel with its parent.
|
|
exit;
|
|
}
|
|
}
|
|
|
|
# detection of cluster or multi-core machines
|
|
|
|
sub detect_machine {
|
|
my ($hostname,$list) = @_;
|
|
$list =~ s/\s+/ /;
|
|
$list =~ s/^ //;
|
|
$list =~ s/ $//;
|
|
foreach my $machine (split(/ /,$list)) {
|
|
return 1 if $hostname =~ /$machine/;
|
|
}
|
|
return 0;
|
|
}
|
|
|
|
sub detect_if_cluster {
|
|
my $hostname = `hostname`; chop($hostname);
|
|
foreach my $line (`cat $RealBin/experiment.machines`) {
|
|
next unless $line =~ /^cluster: (.+)$/;
|
|
if (&detect_machine($hostname,$1)) {
|
|
$CLUSTER = 1;
|
|
print "running on a cluster\n" if $CLUSTER;
|
|
}
|
|
}
|
|
}
|
|
|
|
sub detect_if_multicore {
|
|
my $hostname = `hostname`; chop($hostname);
|
|
foreach my $line (`cat $RealBin/experiment.machines`) {
|
|
next unless $line =~ /^multicore-(\d+): (.+)$/;
|
|
my ($cores,$list) = ($1,$2);
|
|
if (&detect_machine($hostname,$list)) {
|
|
$MAX_ACTIVE = $cores;
|
|
$MULTICORE = 1;
|
|
}
|
|
}
|
|
}
|
|
|
|
### Read the meta information about all possible steps
|
|
|
|
sub read_meta {
|
|
open(META,$META) || die("ERROR: no meta file at $META");
|
|
my ($module,$step);
|
|
while(<META>) {
|
|
s/\#.*$//; # strip comments
|
|
next if /^\s*$/;
|
|
while (/\\\s*$/) {
|
|
$_ .= <META>;
|
|
s/\s*\\\s*[\n\r]*\s+/ /;
|
|
}
|
|
if (/^\[(.+)\]\s+(\S+)/) {
|
|
$module = $1;
|
|
push @MODULE,$module;
|
|
$MODULE_TYPE{$module} = $2;
|
|
# print "MODULE_TYPE{$module} = $2;\n";
|
|
}
|
|
elsif (/^(\S+)/) {
|
|
$step = $1;
|
|
push @{$MODULE_STEP{$module}},$step;
|
|
# print "{MODULE_STEP{$module}},$step;\n";
|
|
}
|
|
elsif (/^\s+(\S+): (.+\S)\s*$/) {
|
|
if ($1 eq "in") {
|
|
@{$STEP_IN{"$module:$step"}} = split(/\s+/,$2);
|
|
}
|
|
elsif ($1 eq "out") {
|
|
$STEP_OUT{"$module:$step"} = $2;
|
|
}
|
|
elsif ($1 eq "default-name") {
|
|
$STEP_OUTNAME{"$module:$step"} = $2;
|
|
}
|
|
elsif ($1 eq "tmp-name") {
|
|
$STEP_TMPNAME{"$module:$step"} = $2;
|
|
}
|
|
elsif ($1 eq "final-model") {
|
|
$STEP_FINAL{"$module:$step"} = $2;
|
|
}
|
|
elsif ($1 eq "pass-unless") {
|
|
@{$STEP_PASS{"$module:$step"}} = split(/\s+/,$2);
|
|
push @{$RERUN_ON_CHANGE{"$module:$step"}}, split(/\s+/,$2);
|
|
}
|
|
elsif ($1 eq "pass-if") {
|
|
@{$STEP_PASS_IF{"$module:$step"}} = split(/\s+/,$2);
|
|
push @{$RERUN_ON_CHANGE{"$module:$step"}}, split(/\s+/,$2);
|
|
}
|
|
elsif ($1 eq "ignore-unless") {
|
|
$STEP_IGNORE{"$module:$step"} = $2;
|
|
}
|
|
elsif ($1 eq "ignore-if") {
|
|
$STEP_IGNORE_IF{"$module:$step"} = $2;
|
|
}
|
|
elsif ($1 eq "qsub-script") {
|
|
$QSUB_SCRIPT{"$module:$step"}++;
|
|
}
|
|
elsif ($1 eq "rerun-on-change") {
|
|
push @{$RERUN_ON_CHANGE{"$module:$step"}}, split(/\s+/,$2);
|
|
}
|
|
elsif ($1 eq "only-existence-matters") {
|
|
$ONLY_EXISTENCE_MATTERS{"$module:$step"}{$2}++;
|
|
}
|
|
elsif ($1 eq "multiref") {
|
|
$MULTIREF{"$module:$step"} = $2;
|
|
}
|
|
elsif ($1 eq "template") {
|
|
my $escaped_template = $2;
|
|
$escaped_template =~ s/^IN/EMS_IN_EMS/;
|
|
$escaped_template =~ s/ IN(\d*)/ EMS_IN$1_EMS/g;
|
|
$escaped_template =~ s/ OUT/ EMS_OUT_EMS/g;
|
|
$escaped_template =~ s/TMP/EMS_TMP_EMS/g;
|
|
$TEMPLATE{"$module:$step"} = $escaped_template;
|
|
}
|
|
elsif ($1 eq "template-if") {
|
|
my $escaped_template = $2;
|
|
$escaped_template =~ s/^IN/EMS_IN_EMS/;
|
|
$escaped_template =~ s/ IN(\d*)/ EMS_IN$1_EMS/g;
|
|
$escaped_template =~ s/ OUT/ EMS_OUT_EMS/g;
|
|
$escaped_template =~ s/TMP/EMS_TMP_EMS/g;
|
|
my @IF = split(/\s+/,$escaped_template);
|
|
push @{$TEMPLATE_IF{"$module:$step"}}, \@IF;
|
|
}
|
|
elsif ($1 eq "parallelizable") {
|
|
$PARALLELIZE{"$module:$step"}++;
|
|
}
|
|
elsif ($1 eq "only-factor-0") {
|
|
$ONLY_FACTOR_0{"$module:$step"}++;
|
|
}
|
|
elsif ($1 eq "error") {
|
|
push @{$ERROR{"$module:$step"}}, $2;
|
|
}
|
|
elsif ($1 eq "not-error") {
|
|
push @{$NOT_ERROR{"$module:$step"}}, $2;
|
|
}
|
|
else {
|
|
die("META ERROR unknown parameter: $1");
|
|
}
|
|
}
|
|
else {
|
|
die("META ERROR buggy line $_");
|
|
}
|
|
}
|
|
close(META);
|
|
}
|
|
|
|
### Read the configuration file
|
|
|
|
sub read_config {
|
|
# read the file
|
|
my $module = "GENERAL";
|
|
my $error = 0;
|
|
my $ignore = 0;
|
|
my $line_count=0;
|
|
open(INI,$CONFIG_FILE) || die("ERROR: CONFIG FILE NOT FOUND: $CONFIG_FILE");
|
|
while(<INI>) {
|
|
$line_count++;
|
|
s/\#.*$//; # strip comments
|
|
next if /^\#/ || /^\s*$/;
|
|
while (/\\\s*$/) { # merge with next line
|
|
s/\s*\\\s*$/ /;
|
|
$_ .= <INI>;
|
|
}
|
|
if (/^\[(.+)\]/) {
|
|
$module = $1;
|
|
$ignore = /ignore/i;
|
|
push @MODULE_LIST,$1 unless $ignore;
|
|
}
|
|
elsif (! $ignore) {
|
|
if (/^(\S+) = (.+)$/) {
|
|
my $parameter = $1;
|
|
my $value = $2;
|
|
$value =~ s/\s+/ /g;
|
|
$value =~ s/^ //;
|
|
$value =~ s/ $//;
|
|
my @VALUE;
|
|
if ($value =~ /^\"(.*)\"$/) {
|
|
@VALUE = ($1);
|
|
}
|
|
else {
|
|
@VALUE = split(/ /,$value);
|
|
}
|
|
$CONFIG{"$module:$parameter"} = \@VALUE;
|
|
}
|
|
else {
|
|
print STDERR "BUGGY CONFIG LINE ($line_count): $_";
|
|
$error++;
|
|
}
|
|
}
|
|
}
|
|
die("$error ERROR".(($error>1)?"s":"")." IN CONFIG FILE") if $error;
|
|
|
|
# resolve parameters used in values
|
|
my $resolve = 1;
|
|
my $loop_count = 0;
|
|
while($resolve && $loop_count++ < 100) {
|
|
$resolve = 0;
|
|
foreach my $parameter (keys %CONFIG) {
|
|
foreach (@{$CONFIG{$parameter}}) {
|
|
next unless /\$/;
|
|
my $escaped = 0;
|
|
die ("BAD USE OF \$ IN VALUE used in parameter $parameter")
|
|
if ! ( /^(.*)\$([a-z\-\:\d]+)(.*)$/i ||
|
|
(/^(.*)\$\{([a-z\-\:\d]+)\}(.*)$/i && ($escaped = 1)));
|
|
my ($pre,$substitution,$post) = ($1,$2,$3);
|
|
my $pattern = $substitution;
|
|
if ($substitution !~ /\:/) { # handle local variables
|
|
$parameter =~ /^(.+)\:/;
|
|
$substitution = $1.":".$substitution;
|
|
}
|
|
|
|
my $orig = $substitution;
|
|
$substitution =~ s/^(.+):.+:(.+)$/$1:$2/ # not set-specific
|
|
unless defined($CONFIG{$substitution});
|
|
$substitution = "GENERAL:$2" # back off to general
|
|
unless defined($CONFIG{$substitution});
|
|
die ("UNKNOWN PARAMETER $orig used in parameter $parameter")
|
|
unless defined($CONFIG{$substitution});
|
|
|
|
my $o = $CONFIG{$substitution}[0];
|
|
print "changing $_ to " if $VERBOSE;
|
|
s/\$\{$pattern\}/$o/ if $escaped;
|
|
s/\$$pattern/$o/ unless $escaped;
|
|
print "$_\n" if $VERBOSE;
|
|
if (/\$/) {
|
|
print "more resolving needed\n" if $VERBOSE;
|
|
$resolve = 1;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
close(INI);
|
|
die("ERROR: CIRCULAR PARAMETER DEFINITION") if $resolve;
|
|
|
|
# check if specified files exist
|
|
$error = 0;
|
|
foreach my $parameter (keys %CONFIG) {
|
|
foreach (@{$CONFIG{$parameter}}) {
|
|
next if $parameter =~ /temp-dir/;
|
|
next if (!/^\// || -e); # ok if not file, or exists
|
|
my $file = $_;
|
|
$file =~ s/ .+$//; # remove switches
|
|
my $gz = $file; $gz =~ s/\.gz$//;
|
|
next if -e $gz; # ok if non gzipped exists
|
|
next if `find $file* -maxdepth 0 -follow`; # ok if stem
|
|
print STDERR "$parameter: file $_ does not exist!\n";
|
|
$error++;
|
|
}
|
|
}
|
|
die if $error;
|
|
}
|
|
|
|
# log parameter settings into a file
|
|
|
|
sub log_config {
|
|
my $dir = &check_and_get("GENERAL:working-dir");
|
|
`mkdir -p $dir/steps`;
|
|
my $config_file = &steps_file("config.$VERSION",$VERSION);
|
|
`cp $CONFIG_FILE $config_file` unless $CONTINUE;
|
|
open(PARAMETER,">".&steps_file("parameter.$VERSION",$VERSION)) or die "Cannot open: $!";
|
|
foreach my $parameter (sort keys %CONFIG) {
|
|
print PARAMETER "$parameter =";
|
|
foreach (@{$CONFIG{$parameter}}) {
|
|
print PARAMETER " ".$_;
|
|
}
|
|
print PARAMETER "\n";
|
|
}
|
|
close(PARAMETER);
|
|
}
|
|
|
|
### find steps to run
|
|
|
|
sub find_steps {
|
|
# find final output to be produced by the experiment
|
|
if (defined($FINAL_OUT)) {
|
|
push @{$NEEDED{$FINAL_OUT}}, "final";
|
|
}
|
|
elsif (!defined($FINAL_STEP)) {
|
|
push @{$NEEDED{"REPORTING:report"}}, "final";
|
|
}
|
|
|
|
# go through each module
|
|
while(1) {
|
|
my $step_count_before = scalar(@DO_STEP);
|
|
for(my $m=$#MODULE; $m>=0; $m--) {
|
|
my $module = $MODULE[$m];
|
|
|
|
# if module is "multiple" go through each set
|
|
if ($MODULE_TYPE{$module} eq "multiple") {
|
|
my @SETS = &get_sets($module);
|
|
foreach my $set (@SETS) {
|
|
&find_steps_for_module($module,$set);
|
|
}
|
|
}
|
|
|
|
# if module is "synchronous" go through each set of previous
|
|
elsif ($MODULE_TYPE{$module} eq "synchronous") {
|
|
my $previous_module = $MODULE[$m-1];
|
|
my @SETS = &get_sets($previous_module);
|
|
foreach my $set (@SETS) {
|
|
&find_steps_for_module($module,$set);
|
|
}
|
|
}
|
|
|
|
# otherwise, execute module once
|
|
else {
|
|
&find_steps_for_module($module,"");
|
|
}
|
|
}
|
|
last if $step_count_before == scalar(@DO_STEP);
|
|
}
|
|
}
|
|
|
|
sub find_steps_for_module {
|
|
my ($module,$set,$final_module) = @_;
|
|
|
|
print "processing module $module:$set\n" if $VERBOSE;
|
|
|
|
# go through potential steps from last to first (counter-chronological)
|
|
foreach my $stepname (reverse @{$MODULE_STEP{$module}}) {
|
|
|
|
my $step = &construct_name($module,$set,$stepname);
|
|
my $defined_step = &defined_step($step); # without set
|
|
next if defined($STEP_LOOKUP{$step});
|
|
|
|
# FIRST, some checking...
|
|
print "\tchecking step: $step\n" if $VERBOSE;
|
|
|
|
# only add this step, if its output is needed by another step
|
|
my $out = &construct_name($module,$set,$STEP_OUT{$defined_step});
|
|
print "\t\tproduces $out\n" if $VERBOSE;
|
|
next unless defined($NEEDED{$out}) || (defined($FINAL_STEP) && $FINAL_STEP eq $step);
|
|
print "\t\tneeded\n" if $VERBOSE;
|
|
|
|
# if output of a step is specified, you do not have
|
|
# to execute that step
|
|
if(defined($CONFIG{$out})) {
|
|
$GIVEN{$out} = $step;
|
|
next;
|
|
}
|
|
print "\t\toutput not specified in config\n" if $VERBOSE;
|
|
|
|
# not needed, if optional and not specified
|
|
if (defined($STEP_IGNORE{$defined_step})) {
|
|
my $next = 0;
|
|
my $and = 0;
|
|
my @IGNORE = split(/ /,$STEP_IGNORE{$defined_step});
|
|
if ($IGNORE[0] eq "AND") {
|
|
$and = 1;
|
|
shift @IGNORE;
|
|
}
|
|
foreach my $ignore (@IGNORE) {
|
|
my $extended_name = &extend_local_name($module,$set,$ignore);
|
|
if (! &backoff_and_get($extended_name)) {
|
|
print "\t\tignored because of non-existance of ".$extended_name."\n" if $VERBOSE;
|
|
$next++;
|
|
}
|
|
}
|
|
next if !$and && ($next == scalar @IGNORE); # OR: all parameters have to be missing
|
|
next if $and && $next; # AND: any parameter has to be missing
|
|
print "\t\t=> not all non-existant, not ignored" if $next && $VERBOSE;
|
|
}
|
|
|
|
# not needed, if alternative step is specified
|
|
if (defined($STEP_IGNORE_IF{$defined_step})) {
|
|
my $next = 0;
|
|
foreach my $ignore (split(/ /,$STEP_IGNORE_IF{$defined_step})) {
|
|
my $extended_name = &extend_local_name($module,$set,$ignore);
|
|
if (&backoff_and_get($extended_name)) {
|
|
print "\t\tignored because of existance of ".$extended_name."\n" if $VERBOSE;
|
|
$next++;
|
|
}
|
|
}
|
|
next if $next;
|
|
}
|
|
|
|
# OK, add step to the list
|
|
|
|
push @DO_STEP,$step;
|
|
$STEP_LOOKUP{$step} = $#DO_STEP;
|
|
print "\tdo-step: $step\n" if $VERBOSE;
|
|
|
|
# mark as pass step (where no action is taken), if step is
|
|
# optional and nothing needs to be do done
|
|
if (defined($STEP_PASS{$defined_step})) {
|
|
my $flag = 1;
|
|
foreach my $pass (@{$STEP_PASS{$defined_step}}) {
|
|
$flag = 0
|
|
if &backoff_and_get(&extend_local_name($module,$set,$pass));
|
|
}
|
|
$PASS{$#DO_STEP}++ if $flag;
|
|
}
|
|
|
|
if (defined($STEP_PASS_IF{$defined_step})) {
|
|
my $flag = 0;
|
|
foreach my $pass (@{$STEP_PASS_IF{$defined_step}}) {
|
|
$flag = 1
|
|
if &backoff_and_get(&extend_local_name($module,$set,$pass));
|
|
}
|
|
$PASS{$#DO_STEP}++ if $flag;
|
|
}
|
|
|
|
# special case for passing: steps that only affect factor 0
|
|
if (defined($ONLY_FACTOR_0{$defined_step})) {
|
|
my $FACTOR = &backoff_and_get_array("LM:$set:factors");
|
|
if (defined($FACTOR)) {
|
|
my $ok = 0;
|
|
foreach my $factor (@{$FACTOR}) {
|
|
$ok++ if ($factor eq "word");
|
|
}
|
|
$PASS{$#DO_STEP}++ unless $ok;
|
|
}
|
|
}
|
|
|
|
# check for dependencies
|
|
foreach (@{$STEP_IN{$defined_step}}) {
|
|
my $in = $_;
|
|
|
|
# if multiple potential inputs, find first that matches
|
|
if ($in =~ /=OR=/) {
|
|
my @POTENTIAL_IN = split(/=OR=/,$in);
|
|
foreach my $potential_in (@POTENTIAL_IN) {
|
|
if (&check_producability($module,$set,$potential_in)) {
|
|
$in = $potential_in;
|
|
last;
|
|
}
|
|
|
|
}
|
|
#die("ERROR: none of potential inputs $in possible for $step")
|
|
$in = $POTENTIAL_IN[$#POTENTIAL_IN] if $in =~ /=OR=/;
|
|
}
|
|
|
|
# define input(s) as needed by this step
|
|
my @IN = &construct_input($module,$set,$in);
|
|
foreach my $in (@IN) {
|
|
print "\t\tneeds input $in: " if $VERBOSE;
|
|
if(defined($CONFIG{$in}) && $CONFIG{$in}[0] =~ /^\[(.+)\]$/) {
|
|
# multiple input, explicitly defined (example: LM:{europarl,nc}:lm )
|
|
if ($CONFIG{$in}[0] =~ /^\[([^:]+):{(\S+)}:(\S+)\]$/) {
|
|
my @SETS = split(',', $2);
|
|
foreach my $set (@SETS) {
|
|
$in = &construct_name($1,$set,$3);
|
|
print $in if $VERBOSE;
|
|
push @{$NEEDED{$in}}, $#DO_STEP;
|
|
push @{$USES_INPUT{$#DO_STEP}},$in;
|
|
print "\n\t\tcross-directed to $in\n" if $VERBOSE;
|
|
}
|
|
$in = "";
|
|
}
|
|
else {
|
|
$in = $1;
|
|
print $in if $VERBOSE;
|
|
push @{$NEEDED{$in}}, $#DO_STEP;
|
|
print "\n\t\tcross-directed to $in\n" if $VERBOSE;
|
|
}
|
|
}
|
|
elsif(defined($CONFIG{$in})) {
|
|
print "\n\t\t... but that is specified\n" if $VERBOSE;
|
|
}
|
|
else {
|
|
push @{$NEEDED{$in}}, $#DO_STEP;
|
|
print "\n" if $VERBOSE;
|
|
}
|
|
push @{$USES_INPUT{$#DO_STEP}},$in;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
sub check_producability {
|
|
my ($module,$set,$output) = @_;
|
|
|
|
# find $output requested as input by step in $module/$set
|
|
my @OUT = &construct_input($module,$set,$output);
|
|
|
|
# if multiple outputs (due to multiple sets merged into one),
|
|
# only one needs to exist
|
|
foreach my $out (@OUT) {
|
|
print "producable? $out\n" if $VERBOSE;
|
|
|
|
# producable, if specified as file in the command line
|
|
return 1 if defined($CONFIG{$out});
|
|
|
|
# find defined step that produces this
|
|
$out =~ s/:.+:/:/g;
|
|
my $defined_step;
|
|
foreach my $ds (keys %STEP_OUT) {
|
|
my ($ds_module) = &deconstruct_name($ds);
|
|
my $ds_out = &construct_name($ds_module,"",$STEP_OUT{$ds});
|
|
print "checking $ds -> $ds_out\n" if $VERBOSE;
|
|
$defined_step = $ds if $out eq $ds_out;
|
|
}
|
|
die("ERROR: cannot possibly produce output $out")
|
|
unless $defined_step;
|
|
|
|
# producable, if cannot be ignored
|
|
return 1 unless defined($STEP_IGNORE{$defined_step});
|
|
|
|
# producable, if required parameter specified
|
|
foreach my $ignore (split(/ /,$STEP_IGNORE{$defined_step})) {
|
|
my ($ds_module) = &deconstruct_name($defined_step);
|
|
my $ds_set = $set;
|
|
$ds_set = "" if $MODULE_TYPE{$ds_module} eq "single";
|
|
my $req = &construct_name($ds_module,$ds_set,$ignore);
|
|
print "producable req $req\n" if $VERBOSE;
|
|
return 1 if defined($CONFIG{$req});
|
|
}
|
|
}
|
|
print "not producable: ($module,$set,$output)\n" if $VERBOSE;
|
|
return 0;
|
|
}
|
|
|
|
# given a current module and set, expand the input definition
|
|
# into actual input file parameters
|
|
sub construct_input {
|
|
my ($module,$set,$in) = @_;
|
|
|
|
# potentially multiple input files
|
|
my @IN;
|
|
|
|
# input from same module
|
|
if ($in !~ /([^:]+):(\S+)/) {
|
|
push @IN, &construct_name($module,$set,$in);
|
|
}
|
|
|
|
# input from previous model, multiple
|
|
elsif ($MODULE_TYPE{$1} eq "multiple") {
|
|
my @SETS = &get_sets($1);
|
|
foreach my $set (@SETS) {
|
|
push @IN, &construct_name($1,$set,$2);
|
|
}
|
|
}
|
|
# input from previous model, synchronized to multiple
|
|
elsif ($1 eq "EVALUATION" && $module eq "REPORTING") {
|
|
my @SETS = &get_sets("EVALUATION");
|
|
foreach my $set (@SETS) {
|
|
push @IN, &construct_name($1,$set,$2);
|
|
}
|
|
}
|
|
# input from previous module, single (ignore current set)
|
|
else {
|
|
push @IN,$in;
|
|
}
|
|
|
|
return @IN;
|
|
}
|
|
|
|
# get the set names for a module that runs on multiple sets
|
|
# (e.g. multiple LMs, multiple training corpora, multiple test sets)
|
|
sub get_sets {
|
|
my ($config) = @_;
|
|
my @SET;
|
|
foreach (@MODULE_LIST) {
|
|
if (/^$config:([^:]+)/) {
|
|
push @SET,$1;
|
|
}
|
|
}
|
|
return @SET;
|
|
}
|
|
|
|
# DELETION OF STEPS AND VERSIONS
|
|
# delete step files for steps that have crashed
|
|
sub delete_crashed {
|
|
my $crashed = 0;
|
|
for(my $i=0;$i<=$#DO_STEP;$i++) {
|
|
my $step_file = &versionize(&step_file($i),$DELETE_CRASHED);
|
|
next unless -e $step_file;
|
|
if (! -e $step_file.".DONE" || # interrupted (machine went down)
|
|
&check_if_crashed($i,$DELETE_CRASHED,"no wait")) { # noted crash
|
|
&delete_step($DO_STEP[$i],$DELETE_CRASHED);
|
|
$crashed++;
|
|
}
|
|
}
|
|
print "run with -exec to delete steps\n" if $crashed && !$EXECUTE;
|
|
print "nothing to do\n" unless $crashed;
|
|
}
|
|
|
|
# delete all step and data files for a version
|
|
sub delete_version {
|
|
|
|
# check which versions are already deleted
|
|
my %ALREADY_DELETED;
|
|
my $dir = &check_and_get("GENERAL:working-dir");
|
|
open(VERSION,"ls $dir/steps/*/deleted.* 2>/dev/null|");
|
|
while(<VERSION>) {
|
|
/deleted\.(\d+)/;
|
|
$ALREADY_DELETED{$1}++;
|
|
}
|
|
close(VERSION);
|
|
|
|
# check if any of the steps are re-used by other versions
|
|
my (%USED_BY_OTHERS,%DELETABLE,%NOT_DELETABLE);
|
|
open(VERSION,"ls $dir/steps|");
|
|
while(my $version = <VERSION>) {
|
|
chop($version);
|
|
next if $version !~ /^\d+/ || $version == 0;
|
|
open(RE_USE,"steps/$version/re-use.$version");
|
|
while(<RE_USE>) {
|
|
next unless /^(.+) (\d+)$/;
|
|
my ($step,$re_use_version) = ($1,$2);
|
|
|
|
# a step in the current version that is used in other versions
|
|
$USED_BY_OTHERS{$step}++ if $re_use_version == $DELETE_VERSION && !defined($ALREADY_DELETED{$version});
|
|
|
|
# potentially deletable step in already deleted version that current version uses
|
|
push @{$DELETABLE{$re_use_version}}, $step if $version == $DELETE_VERSION && defined($ALREADY_DELETED{$re_use_version});
|
|
|
|
# not deletable step used by not-deleted version
|
|
$NOT_DELETABLE{$re_use_version}{$step}++ if $version != $DELETE_VERSION && !defined($ALREADY_DELETED{$version});
|
|
}
|
|
close(RE_USE);
|
|
}
|
|
|
|
# go through all steps for which step files where created
|
|
open(STEPS,"ls $dir/steps/$DELETE_VERSION/[A-Z]*.$DELETE_VERSION|");
|
|
while(my $step_file = <STEPS>) {
|
|
chomp($step_file);
|
|
my $step = &get_step_from_step_file($step_file);
|
|
next if $USED_BY_OTHERS{$step};
|
|
&delete_step($step,$DELETE_VERSION);
|
|
}
|
|
|
|
# orphan killing: delete steps in deleted versions, if they were only preserved because this version needed them
|
|
foreach my $version (keys %DELETABLE) {
|
|
foreach my $step (@{$DELETABLE{$version}}) {
|
|
next if defined($NOT_DELETABLE{$version}) && defined($NOT_DELETABLE{$version}{$step});
|
|
&delete_step($step,$version);
|
|
}
|
|
}
|
|
my $deleted_flag_file = &steps_file("deleted.$DELETE_VERSION",$DELETE_VERSION);
|
|
`touch $deleted_flag_file` if $EXECUTE;
|
|
}
|
|
|
|
sub get_step_from_step_file {
|
|
my ($step) = @_;
|
|
$step =~ s/^.+\///;
|
|
$step =~ s/\.\d+$//;
|
|
$step =~ s/_/:/g;
|
|
return $step;
|
|
}
|
|
|
|
sub delete_step {
|
|
my ($step_name,$version) = @_;
|
|
my ($module,$set,$step) = &deconstruct_name($step_name);
|
|
|
|
my $step_file = &versionize(&step_file2($module,$set,$step),$version);
|
|
print "delete step $step_file\n";
|
|
`rm $step_file $step_file.*` if $EXECUTE;
|
|
|
|
my $out_file = $STEP_OUTNAME{"$module:$step"};
|
|
$out_file =~ s/^(.+\/)([^\/]+)$/$1$set.$2/g if $set;
|
|
&delete_output(&versionize(&long_file_name($out_file,$module,$set), $version));
|
|
|
|
if (defined($STEP_TMPNAME{"$module:$step"})) {
|
|
my $tmp_file = &get_tmp_file($module,$set,$step,$version);
|
|
&delete_output($tmp_file);
|
|
}
|
|
}
|
|
|
|
# delete output files that match a given prefix
|
|
sub delete_output {
|
|
my ($file) = @_;
|
|
# delete directory that matches exactly
|
|
if (-d $file) {
|
|
print "\tdelete directory $file\n";
|
|
`rm -r $file` if $EXECUTE;
|
|
}
|
|
# delete regular file that matches exactly
|
|
elsif (-e $file) {
|
|
print "\tdelete file $file\n";
|
|
`rm $file` if $EXECUTE;
|
|
}
|
|
# delete files that have additional extension
|
|
$file =~ /^(.+)\/([^\/]+)$/;
|
|
my ($dir,$f) = ($1,$2);
|
|
my @FILES = `ls $file.* 2>/dev/null`;
|
|
foreach (`ls $dir`) {
|
|
chop;
|
|
next unless substr($_,0,length($f)) eq $f;
|
|
if (-d "$dir/$_") {
|
|
print "\tdelete directory $file\n";
|
|
`rm -r $dir/$_` if $EXECUTE;
|
|
}
|
|
elsif (-e "$dir/$_") {
|
|
print "\tdelete file $dir/$_\n";
|
|
`rm $dir/$_` if $EXECUTE;
|
|
}
|
|
}
|
|
}
|
|
|
|
# RE-USE
|
|
# look for completed step jobs from previous experiments
|
|
sub find_re_use {
|
|
my $dir = &check_and_get("GENERAL:working-dir");
|
|
return unless -e "$dir/steps";
|
|
|
|
for(my $i=0;$i<=$#DO_STEP;$i++) {
|
|
%{$RE_USE[$i]} = ();
|
|
}
|
|
|
|
# find older steps from previous versions that can be re-used
|
|
open(LS,"find $dir/steps/* -maxdepth 1 -follow | sort -r |");
|
|
while(my $info_file = <LS>) {
|
|
next unless $info_file =~ /INFO$/;
|
|
$info_file =~ s/.+\/([^\/]+)$/$1/; # ignore path
|
|
for(my $i=0;$i<=$#DO_STEP;$i++) {
|
|
# next if $RE_USE[$i]; # already found one
|
|
my $pattern = &step_file($i);
|
|
$pattern =~ s/\+/\\+/; # escape plus signs in file names
|
|
$pattern = "^$pattern.(\\d+).INFO\$";
|
|
$pattern =~ s/.+\/([^\/]+)$/$1/; # ignore path
|
|
next unless $info_file =~ /$pattern/;
|
|
my $old_version = $1;
|
|
print "re_use $i $DO_STEP[$i] (v$old_version) ".join(" ",keys %{$RE_USE[$i]})." ?\n" if $VERBOSE;
|
|
print "\tno info file ".&versionize(&step_file($i),$old_version).".INFO\n" if ! -e &versionize(&step_file($i),$old_version).".INFO" && $VERBOSE;
|
|
print "\tno done file " if ! -e &versionize(&step_file($i),$old_version).".DONE" && $VERBOSE;
|
|
if (! -e &versionize(&step_file($i),$old_version).".INFO") {
|
|
print "\tinfo file does not exist\n" if $VERBOSE;
|
|
print "\tnot re-usable\n" if $VERBOSE;
|
|
}
|
|
elsif (! -e &versionize(&step_file($i),$old_version).".DONE") {
|
|
print "\tstep not done (done file does not exist)\n" if $VERBOSE;
|
|
print "\tnot re-usable\n" if $VERBOSE;
|
|
}
|
|
elsif (! &check_info($i,$old_version) ) {
|
|
print "\tparameters from info file do not match\n" if $VERBOSE;
|
|
print "\tnot re-usable\n" if $VERBOSE;
|
|
}
|
|
elsif (&check_if_crashed($i,$old_version)) {
|
|
print "\tstep crashed\n" if $VERBOSE;
|
|
print "\tnot re-usable\n" if $VERBOSE;
|
|
}
|
|
else {
|
|
$RE_USE[$i]{$old_version}++;
|
|
print "\tre-usable\n" if $VERBOSE;
|
|
}
|
|
}
|
|
}
|
|
close(LS);
|
|
|
|
# all preceding steps have to be re-usable
|
|
# otherwise output from old step can not be re-used
|
|
my $change = 1;
|
|
while($change) {
|
|
$change = 0;
|
|
|
|
for(my $i=0;$i<=$#DO_STEP;$i++) {
|
|
next unless $RE_USE[$i];
|
|
foreach my $run (keys %{$RE_USE[$i]}) {
|
|
print "check on dependencies for $i ($run) $DO_STEP[$i]\n" if $VERBOSE;
|
|
foreach (@{$DEPENDENCY[$i]}) {
|
|
my $parent = $_;
|
|
print "\tchecking on $parent $DO_STEP[$parent]\n" if $VERBOSE;
|
|
my @PASSING;
|
|
# skip steps that are passed
|
|
while (defined($PASS{$parent})) {
|
|
if (scalar (@{$DEPENDENCY[$parent]}) == 0) {
|
|
$parent = 0;
|
|
print "\tprevious step's output is specified\n" if $VERBOSE;
|
|
}
|
|
else {
|
|
push @PASSING, $parent;
|
|
$parent = $DEPENDENCY[$parent][0];
|
|
print "\tmoving up to $parent $DO_STEP[$parent]\n" if $VERBOSE;
|
|
}
|
|
}
|
|
# check if parent step may be re-used
|
|
if ($parent) {
|
|
my $reuse_run = $run;
|
|
# if recursive re-use, switch to approapriate run
|
|
if (defined($RECURSIVE_RE_USE{$i,$run,$DO_STEP[$parent]})) {
|
|
print "\trecursive re-use run $reuse_run\n" if $VERBOSE;
|
|
$reuse_run = $RECURSIVE_RE_USE{$i,$run,$DO_STEP[$parent]};
|
|
}
|
|
# additional check for straight re-use
|
|
else {
|
|
# re-use step has to have passed the same steps
|
|
foreach (@PASSING) {
|
|
my $passed = $DO_STEP[$_];
|
|
$passed =~ s/:/_/g;
|
|
if (-e &steps_file("$passed.$run",$run)) {
|
|
delete($RE_USE[$i]{$run});
|
|
$change = 1;
|
|
print "\tpassed step $DO_STEP[$_] used in re-use run $run -> fail\n" if $VERBOSE;
|
|
}
|
|
}
|
|
}
|
|
# re-use step has to exist for this run
|
|
if (! defined($RE_USE[$parent]{$reuse_run})) {
|
|
print "\tno previous step -> fail\n" if $VERBOSE;
|
|
delete($RE_USE[$i]{$run});
|
|
$change = 1;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
# summarize and convert hashes into integers for to be re-used
|
|
print "\nSTEP SUMMARY:\n";
|
|
open(RE_USE,">".&steps_file("re-use.$VERSION",$VERSION)) or die "Cannot open: $!";
|
|
for(my $i=$#DO_STEP;$i>=0;$i--) {
|
|
if ($PASS{$i}) {
|
|
$RE_USE[$i] = 0;
|
|
next;
|
|
}
|
|
print "$i $DO_STEP[$i] ->\t";
|
|
if (scalar(keys %{$RE_USE[$i]})) {
|
|
my @ALL = sort { $a <=> $b} keys %{$RE_USE[$i]};
|
|
print "re-using (".join(" ",@ALL).")\n";
|
|
$RE_USE[$i] = $ALL[0];
|
|
if ($ALL[0] != $VERSION) {
|
|
print RE_USE "$DO_STEP[$i] $ALL[0]\n";
|
|
}
|
|
}
|
|
else {
|
|
print "run\n";
|
|
$RE_USE[$i] = 0;
|
|
}
|
|
}
|
|
close(RE_USE);
|
|
}
|
|
|
|
sub find_dependencies {
|
|
for(my $i=0;$i<=$#DO_STEP;$i++) {
|
|
@{$DEPENDENCY[$i]} = ();
|
|
}
|
|
for(my $i=0;$i<=$#DO_STEP;$i++) {
|
|
my $step = $DO_STEP[$i];
|
|
$step =~ /^(.+:)[^:]+$/;
|
|
my $module_set = $1;
|
|
foreach my $needed_by (@{$NEEDED{$module_set.$STEP_OUT{&defined_step($step)}}}) {
|
|
print "$needed_by needed by $i\n" if $VERBOSE;
|
|
next if $needed_by eq 'final';
|
|
push @{$DEPENDENCY[$needed_by]},$i;
|
|
}
|
|
}
|
|
|
|
# for(my $i=0;$i<=$#DO_STEP;$i++) {
|
|
# print "to run step $i ($DO_STEP[$i]), we first need to run step(s) ".join(" ",@{$DEPENDENCY[$i]})."\n";
|
|
# }
|
|
}
|
|
|
|
sub draw_agenda_graph {
|
|
my %M;
|
|
my $dir = &check_and_get("GENERAL:working-dir");
|
|
open(DOT,">".&steps_file("graph.$VERSION.dot",$VERSION)) or die "Cannot open: $!";
|
|
print DOT "digraph Experiment$VERSION {\n";
|
|
print DOT " ranksep=0;\n";
|
|
for(my $i=0;$i<=$#DO_STEP;$i++) {
|
|
my $step = $DO_STEP[$i];
|
|
$step =~ /^(.+):[^:]+$/;
|
|
my $module_set = $1;
|
|
push @{$M{$module_set}},$i;
|
|
}
|
|
my $i = 0;
|
|
my (@G,%GIVEN_NUMBER);
|
|
foreach (values %GIVEN) {
|
|
push @G,$_;
|
|
$GIVEN_NUMBER{$_} = $#G;
|
|
/^(.+):[^:]+$/;
|
|
my $module_set = $1;
|
|
push @{$M{$module_set}},"g".($#G);
|
|
}
|
|
my $m = 0;
|
|
foreach my $module (keys %M) {
|
|
print DOT " subgraph cluster_".($m++)." {\n";
|
|
print DOT " fillcolor=\"lightyellow\";\n";
|
|
print DOT " shape=box;\n";
|
|
print DOT " style=filled;\n";
|
|
print DOT " fontsize=10;\n";
|
|
print DOT " label=\"$module\";\n";
|
|
foreach my $i (@{$M{$module}}) {
|
|
if ($i =~ /g(\d+)/) {
|
|
my $step = $G[$1];
|
|
$step =~ /^.+:([^:]+)$/;
|
|
print DOT " $i [label=\"$1\",shape=box,fontsize=10,height=0,style=filled,fillcolor=\"#c0b060\"];\n";
|
|
}
|
|
else {
|
|
my $step = $DO_STEP[$i];
|
|
$step =~ s/^.+:([^:]+)$/$1/;
|
|
$step .= " (".$RE_USE[$i].")" if $RE_USE[$i];
|
|
|
|
my $color = "green";
|
|
$color = "#0000ff" if defined($DO{$i}) && $DO{$i} >= 1;
|
|
$color = "#8080ff" if defined($DONE{$i}) || ($RE_USE[$i] && $RE_USE[$i] == $VERSION);
|
|
$color = "lightblue" if $RE_USE[$i] && $RE_USE[$i] != $VERSION;
|
|
$color = "red" if defined($CRASHED{$i});
|
|
$color = "lightyellow" if defined($PASS{$i});
|
|
|
|
print DOT " $i [label=\"$step\",shape=box,fontsize=10,height=0,style=filled,fillcolor=\"$color\"];\n";
|
|
}
|
|
}
|
|
print DOT " }\n";
|
|
}
|
|
for(my $i=0;$i<=$#DO_STEP;$i++) {
|
|
foreach (@{$DEPENDENCY[$i]}) {
|
|
print DOT " $_ -> $i;\n";
|
|
}
|
|
}
|
|
|
|
# steps that do not have to be performed, because
|
|
# their output is given
|
|
foreach my $out (keys %GIVEN) {
|
|
foreach my $needed_by (@{$NEEDED{$out}}) {
|
|
print DOT " g".$GIVEN_NUMBER{$GIVEN{$out}}." -> $needed_by;\n";
|
|
}
|
|
}
|
|
|
|
print DOT "}\n";
|
|
close(DOT);
|
|
my $graph_file = &steps_file("graph.$VERSION",$VERSION);
|
|
`dot -Tps $graph_file.dot >$graph_file.ps 2>/dev/null`;
|
|
`convert -alpha off $graph_file.ps $graph_file.png`;
|
|
}
|
|
|
|
sub define_step {
|
|
my ($step) = @_;
|
|
my $dir = &check_and_get("GENERAL:working-dir");
|
|
`mkdir -p $dir` if ! -e $dir;
|
|
my @STEP;
|
|
if ($step eq "all") {
|
|
for(my $i=0;$i<=$#DO_STEP;$i++) {
|
|
push @STEP,$i;
|
|
}
|
|
}
|
|
else {
|
|
@STEP = ($step);
|
|
}
|
|
foreach my $i (@STEP) {
|
|
next if $RE_USE[$i];
|
|
next if defined($PASS{$i});
|
|
next if &define_template($i);
|
|
if ($DO_STEP[$i] =~ /^CORPUS:(.+):(post-split-)?factorize$/) {
|
|
&define_corpus_factorize($i);
|
|
}
|
|
elsif ($DO_STEP[$i] eq 'SPLITTER:train') {
|
|
&define_splitter_train($i);
|
|
}
|
|
elsif ($DO_STEP[$i] =~ /^LM:(.+):(post-split-)?factorize$/) {
|
|
&define_lm_factorize($i,$1);
|
|
}
|
|
elsif ($DO_STEP[$i] =~ /^LM:(.+):randomize$/ ||
|
|
$DO_STEP[$i] eq 'INTERPOLATED-LM:randomize') {
|
|
&define_lm_randomize($i,$1);
|
|
}
|
|
elsif ($DO_STEP[$i] =~ /^LM:(.+):train-randomized$/) {
|
|
&define_lm_train_randomized($i,$1);
|
|
}
|
|
elsif ($DO_STEP[$i] =~ /^LM:(.+):train-bilingual-lm$/) {
|
|
&define_lm_train_bilingual_lm($i,$1);
|
|
}
|
|
elsif ($DO_STEP[$i] =~ /^LM:(.+):prepare-bilingual-lm$/) {
|
|
&define_lm_prepare_bilingual_lm($i,$1);
|
|
}
|
|
elsif ($DO_STEP[$i] =~ /^LM:(.+):train-nplm$/) {
|
|
&define_lm_train_nplm($i,$1);
|
|
}
|
|
elsif ($DO_STEP[$i] eq 'TRAINING:prepare-data') {
|
|
&define_training_prepare_data($i);
|
|
}
|
|
elsif ($DO_STEP[$i] eq 'TRAINING:prepare-data-fast-align') {
|
|
&define_training_prepare_data_fast_align($i);
|
|
}
|
|
elsif ($DO_STEP[$i] eq 'TRAINING:run-giza') {
|
|
&define_training_run_giza($i);
|
|
}
|
|
elsif ($DO_STEP[$i] eq 'TRAINING:run-giza-inverse') {
|
|
&define_training_run_giza_inverse($i);
|
|
}
|
|
elsif ($DO_STEP[$i] eq 'TRAINING:symmetrize-giza') {
|
|
&define_training_symmetrize_giza($i);
|
|
}
|
|
elsif ($DO_STEP[$i] eq 'TRAINING:build-biconcor') {
|
|
&define_training_build_biconcor($i);
|
|
}
|
|
elsif ($DO_STEP[$i] eq 'TRAINING:build-suffix-array') {
|
|
&define_training_build_suffix_array($i);
|
|
}
|
|
|
|
elsif ($DO_STEP[$i] eq 'TRAINING:build-lex-trans') {
|
|
&define_training_build_lex_trans($i);
|
|
}
|
|
elsif ($DO_STEP[$i] eq 'TRAINING:extract-phrases') {
|
|
&define_training_extract_phrases($i);
|
|
}
|
|
elsif ($DO_STEP[$i] eq 'TRAINING:build-reordering') {
|
|
&define_training_build_reordering($i);
|
|
}
|
|
elsif ($DO_STEP[$i] eq 'TRAINING:build-ttable') {
|
|
&define_training_build_ttable($i);
|
|
}
|
|
elsif ($DO_STEP[$i] eq 'TRAINING:build-transliteration-model') {
|
|
&define_training_build_transliteration_model($i);
|
|
}
|
|
elsif ($DO_STEP[$i] eq 'TRAINING:build-generation') {
|
|
&define_training_build_generation($i);
|
|
}
|
|
elsif ($DO_STEP[$i] eq 'TRAINING:sigtest-filter-ttable' ||
|
|
$DO_STEP[$i] eq 'TRAINING:sigtest-filter-reordering') {
|
|
&define_training_sigtest_filter($i);
|
|
}
|
|
elsif ($DO_STEP[$i] eq 'TRAINING:create-config' || $DO_STEP[$i] eq 'TRAINING:create-config-interpolated-lm') {
|
|
&define_training_create_config($i);
|
|
}
|
|
elsif ($DO_STEP[$i] =~ /^INTERPOLATED-LM:(post-split-)?factorize-tuning$/) {
|
|
&define_interpolated_lm_factorize_tuning($i);
|
|
}
|
|
elsif ($DO_STEP[$i] eq 'INTERPOLATED-LM:interpolate') {
|
|
&define_interpolated_lm_interpolate($i);
|
|
}
|
|
elsif ($DO_STEP[$i] eq 'INTERPOLATED-LM:binarize' ||
|
|
$DO_STEP[$i] eq 'INTERPOLATED-LM:quantize' ||
|
|
$DO_STEP[$i] eq 'INTERPOLATED-LM:randomize') {
|
|
&define_interpolated_lm_process($i);
|
|
}
|
|
elsif ($DO_STEP[$i] eq 'TUNING:factorize-input') {
|
|
&define_tuningevaluation_factorize($i);
|
|
}
|
|
elsif ($DO_STEP[$i] eq 'TUNING:factorize-input-devtest') {
|
|
&define_tuningevaluation_factorize($i);
|
|
}
|
|
elsif ($DO_STEP[$i] eq 'TUNING:filter') {
|
|
&define_tuningevaluation_filter(undef,$i);
|
|
}
|
|
elsif ($DO_STEP[$i] eq 'TUNING:filter-devtest') {
|
|
&define_tuningevaluation_filter(undef,$i);
|
|
}
|
|
elsif ($DO_STEP[$i] eq 'TUNING:tune') {
|
|
&define_tuning_tune($i);
|
|
}
|
|
elsif ($DO_STEP[$i] =~ /^EVALUATION:(.+):factorize-input$/) {
|
|
&define_tuningevaluation_factorize($i);
|
|
}
|
|
elsif ($DO_STEP[$i] =~ /^EVALUATION:(.+):filter$/) {
|
|
&define_tuningevaluation_filter($1,$i);
|
|
}
|
|
elsif ($DO_STEP[$i] =~ /^EVALUATION:(.+):decode$/) {
|
|
&define_evaluation_decode($1,$i);
|
|
}
|
|
elsif ($DO_STEP[$i] =~ /^EVALUATION:(.+):analysis$/) {
|
|
&define_evaluation_analysis($1,$i);
|
|
}
|
|
elsif ($DO_STEP[$i] =~ /^EVALUATION:(.+):analysis-precision$/) {
|
|
&define_evaluation_analysis_precision($1,$i);
|
|
}
|
|
elsif ($DO_STEP[$i] =~ /^EVALUATION:(.+):analysis-coverage$/) {
|
|
&define_evaluation_analysis_coverage($1,$i);
|
|
}
|
|
elsif ($DO_STEP[$i] =~ /^EVALUATION:(.+):meteor$/) {
|
|
# &define_evaluation_meteor($1);
|
|
}
|
|
elsif ($DO_STEP[$i] =~ /^EVALUATION:(.+):ter$/) {
|
|
# &define_evaluation_ter($1);
|
|
}
|
|
elsif ($DO_STEP[$i] eq 'REPORTING:report') {
|
|
&define_reporting_report($i);
|
|
}
|
|
else {
|
|
print STDERR "ERROR: unknown step $DO_STEP[$i]\n";
|
|
exit;
|
|
}
|
|
}
|
|
}
|
|
|
|
# LOOP that executes the steps
|
|
# including checks, if needed to be executed, waiting for completion, and error detection
|
|
|
|
sub execute_steps {
|
|
my $running_file = &steps_file("running.$VERSION",$VERSION);
|
|
`touch $running_file`;
|
|
|
|
for(my $i=0;$i<=$#DO_STEP;$i++) {
|
|
$DONE{$i}++ if $RE_USE[$i];
|
|
}
|
|
|
|
my $active = 0;
|
|
while(1) {
|
|
|
|
# find steps to be done
|
|
my $repeat_if_passed = 1;
|
|
while($repeat_if_passed) {
|
|
$repeat_if_passed = 0;
|
|
for(my $i=0;$i<=$#DO_STEP;$i++) {
|
|
next if (defined($DONE{$i}));
|
|
next if (defined($DO{$i}));
|
|
next if (defined($CRASHED{$i}));
|
|
my $doable = 1;
|
|
# can't do steps whose predecedents are not done yet
|
|
foreach my $prev_step (@{$DEPENDENCY[$i]}) {
|
|
$doable = 0 if !defined($DONE{$prev_step});
|
|
}
|
|
next unless $doable;
|
|
$DO{$i} = 1;
|
|
|
|
# immediately label pass steps as done
|
|
next unless defined($PASS{$i});
|
|
$DONE{$i} = 1;
|
|
delete($DO{$i});
|
|
$repeat_if_passed = 1;
|
|
}
|
|
}
|
|
|
|
print "number of steps doable or running: ".(scalar keys %DO)." at ".`date`;
|
|
foreach my $step (keys %DO) { print "\t".($DO{$step}==2?"running: ":"doable: ").$DO_STEP[$step]."\n"; }
|
|
return unless scalar keys %DO;
|
|
|
|
# execute new step
|
|
my $done = 0;
|
|
foreach my $i (keys %DO) {
|
|
next unless $DO{$i} == 1;
|
|
if (defined($PASS{$i})) { # immediately label pass steps as done
|
|
$DONE{$i}++;
|
|
delete($DO{$i});
|
|
$done++;
|
|
}
|
|
elsif (! -e &versionize(&step_file($i)).".DONE") {
|
|
my $step = &versionize(&step_file($i));
|
|
&define_step($i);
|
|
&write_info($i);
|
|
|
|
# cluster job submission
|
|
if ($CLUSTER && (!&is_qsub_script($i) || (&backoff_and_get($DO_STEP[$i].":jobs") && (&backoff_and_get($DO_STEP[$i].":jobs")==1)))) {
|
|
$DO{$i}++;
|
|
my $qsub_args = &get_qsub_args($DO_STEP[$i]);
|
|
print "\texecuting $step via qsub $qsub_args ($active active)\n";
|
|
my $qsub_command="qsub $qsub_args -S /bin/bash -e $step.STDERR -o $step.STDOUT $step";
|
|
print "\t$qsub_command\n" if $VERBOSE;
|
|
`$qsub_command`;
|
|
}
|
|
|
|
# execute in fork
|
|
elsif ($CLUSTER || $active < $MAX_ACTIVE) {
|
|
$active++;
|
|
$DO{$i}++;
|
|
print "\texecuting $step via sh ($active active)\n";
|
|
sleep(5);
|
|
if (!fork) {
|
|
`sh $step >$step.STDOUT 2> $step.STDERR`;
|
|
exit;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
# update state
|
|
&draw_agenda_graph() unless $done;
|
|
|
|
# sleep until one more step is done
|
|
while(! $done) {
|
|
sleep($SLEEP);
|
|
my $dir = &check_and_get("GENERAL:working-dir");
|
|
`ls $dir/steps > /dev/null`; # nfs bug
|
|
foreach my $i (keys %DO) {
|
|
if (-e &versionize(&step_file($i)).".DONE") {
|
|
delete($DO{$i});
|
|
if (&check_if_crashed($i)) {
|
|
$CRASHED{$i}++;
|
|
print "step $DO_STEP[$i] crashed\n";
|
|
}
|
|
else {
|
|
$DONE{$i}++;
|
|
}
|
|
$done++;
|
|
$active--;
|
|
}
|
|
}
|
|
`touch $running_file`;
|
|
}
|
|
}
|
|
}
|
|
|
|
# a number of arguments to the job submission may be specified
|
|
# note that this is specific to your gridengine implementation
|
|
# and some options may not work.
|
|
|
|
sub get_qsub_args {
|
|
my ($step) = @_;
|
|
my $qsub_args = &backoff_and_get("$step:qsub-settings");
|
|
$qsub_args = "" unless defined($qsub_args);
|
|
my $memory = &get("$step:qsub-memory");
|
|
$qsub_args .= " -pe memory $memory" if defined($memory);
|
|
my $hours = &get("$step:qsub-hours");
|
|
$qsub_args .= " -l h_rt=$hours:0:0" if defined($hours);
|
|
my $project = &backoff_and_get("$step:qsub-project");
|
|
$qsub_args .= " -P $project" if defined($project);
|
|
$qsub_args =~ s/^ //;
|
|
print "qsub args: $qsub_args\n" if $VERBOSE;
|
|
return $qsub_args;
|
|
}
|
|
|
|
# certain scripts when run on the clusters submit jobs
|
|
# themselves, hence they are executed regularly ("sh script")
|
|
# instead of submited as jobs. here we check for that.
|
|
sub is_qsub_script {
|
|
my ($i) = @_;
|
|
return (defined($QSUB_STEP{$i}) ||
|
|
defined($QSUB_SCRIPT{&defined_step($DO_STEP[$i])}));
|
|
}
|
|
|
|
# write the info file that is consulted to check if
|
|
# a steps has to be redone, even if it was run before
|
|
sub write_info {
|
|
my ($i) = @_;
|
|
my $step = $DO_STEP[$i];
|
|
my $module_set = $step; $module_set =~ s/:[^:]+$//;
|
|
|
|
open(INFO,">".&versionize(&step_file($i)).".INFO") or die "Cannot open: $!";
|
|
my %VALUE = &get_parameters_relevant_for_re_use($i);
|
|
foreach my $parameter (keys %VALUE) {
|
|
print INFO "$parameter = $VALUE{$parameter}\n";
|
|
}
|
|
|
|
# record re-use for recursive re-use
|
|
foreach my $parent (@{$DEPENDENCY[$i]}) {
|
|
my $p = $parent;
|
|
while (defined($PASS{$p}) && scalar @{$DEPENDENCY[$p]}) {
|
|
$p = $DEPENDENCY[$p][0];
|
|
}
|
|
if ($RE_USE[$p]) {
|
|
print INFO "# reuse run $RE_USE[$p] for $DO_STEP[$p]\n";
|
|
}
|
|
}
|
|
|
|
close(INFO);
|
|
}
|
|
|
|
# check the info file...
|
|
sub check_info {
|
|
my ($i,$version) = @_;
|
|
$version = $VERSION unless $version; # default: current version
|
|
my %VALUE = &get_parameters_relevant_for_re_use($i);
|
|
my ($module,$set,$step) = &deconstruct_name($DO_STEP[$i]);
|
|
|
|
my %INFO;
|
|
open(INFO,&versionize(&step_file($i),$version).".INFO") or die "Cannot open: $!";
|
|
while(<INFO>) {
|
|
chop;
|
|
if (/ = /) {
|
|
my ($parameter,$value) = split(/ = /,$_,2);
|
|
$INFO{$parameter} = $value;
|
|
}
|
|
elsif (/^\# reuse run (\d+) for (\S+)/) {
|
|
if ($1>0 && defined($STEP_LOOKUP{$2})) {
|
|
print "\tRECURSIVE_RE_USE{$i,$version,$2} = $1\n" if $VERBOSE;
|
|
$RECURSIVE_RE_USE{$i,$version,$2} = $1;
|
|
}
|
|
else {
|
|
print "\tnot using '$_', step $2 not required\n" if $VERBOSE;
|
|
return 0;
|
|
}
|
|
}
|
|
}
|
|
close(INFO);
|
|
|
|
print "\tcheck parameter count current: ".(scalar keys %VALUE).", old: ".(scalar keys %INFO)."\n" if $VERBOSE;
|
|
return 0 unless scalar keys %INFO == scalar keys %VALUE;
|
|
foreach my $parameter (keys %VALUE) {
|
|
if (! defined($INFO{$parameter})) {
|
|
print "\told has no '$parameter' -> not re-usable\n" if $VERBOSE;
|
|
return 0;
|
|
}
|
|
print "\tcheck '$VALUE{$parameter}' eq '$INFO{$parameter}' -> " if $VERBOSE;
|
|
if (defined($ONLY_EXISTENCE_MATTERS{"$module:$step"}{$parameter})) {
|
|
print "existence ok\n" if $VERBOSE;
|
|
}
|
|
elsif (&match_info_strings($VALUE{$parameter},$INFO{$parameter})) {
|
|
print "ok\n" if $VERBOSE;
|
|
}
|
|
else {
|
|
print "mismatch\n" if $VERBOSE;
|
|
return 0;
|
|
}
|
|
}
|
|
print "\tall parameters match\n" if $VERBOSE;
|
|
return 1;
|
|
}
|
|
|
|
sub match_info_strings {
|
|
my ($current,$old) = @_;
|
|
$current =~ s/ $//;
|
|
$old =~ s/ $//;
|
|
return 1 if $current eq $old;
|
|
# ignore time stamps, if that option is used
|
|
if (defined($IGNORE_TIME)) {
|
|
$current =~ s/\[\d{10}\]//g;
|
|
$old =~ s/\[\d{10}\]//g;
|
|
}
|
|
return 1 if $current eq $old;
|
|
# allowing stars to substitute numbers
|
|
while($current =~ /^([^\*]+)\*(.*)$/) {
|
|
return 0 unless $1 eq substr($old,0,length($1)); # prefix must match
|
|
$current = $2;
|
|
return 0 unless substr($old,length($1)) =~ /^\d+(.*)$/; # must start with number
|
|
$old = $1;
|
|
return 1 if $old eq $current; # done if rest matches
|
|
}
|
|
return 0;
|
|
}
|
|
|
|
sub get_parameters_relevant_for_re_use {
|
|
my ($i) = @_;
|
|
|
|
my %VALUE;
|
|
my $step = $DO_STEP[$i];
|
|
#my $module_set = $step; $module_set =~ s/:[^:]+$//;
|
|
my ($module,$set,$dummy) = &deconstruct_name($step);
|
|
foreach my $parameter (@{$RERUN_ON_CHANGE{&defined_step($step)}}) {
|
|
#if ($parameter =~ /\//) {
|
|
# TODO: handle scripts that need to be checked for time stamps
|
|
#}
|
|
my $value = &backoff_and_get_array(&extend_local_name($module,$set,$parameter));
|
|
$value = join(" ",@{$value}) if ref($value) eq 'ARRAY';
|
|
$VALUE{$parameter} = $value if $value;
|
|
}
|
|
|
|
my ($out,@INPUT) = &get_output_and_input($i);
|
|
my $actually_used = "USED";
|
|
foreach my $in_file (@INPUT) {
|
|
$actually_used .= " ".$in_file;
|
|
}
|
|
$VALUE{"INPUT"} = $actually_used;
|
|
|
|
foreach my $in_file (@{$USES_INPUT{$i}}) {
|
|
my $value = &backoff_and_get($in_file);
|
|
$VALUE{$in_file} = $value if $value;
|
|
}
|
|
|
|
# add timestamp to files
|
|
foreach my $value (values %VALUE) {
|
|
if ($value =~ /^\//) { # file name
|
|
my $file = $value;
|
|
$file =~ s/ .+//; # ignore switches
|
|
if (-e $file) {
|
|
my @filestat = stat($file);
|
|
$value .= " [".$filestat[9]."]";
|
|
}
|
|
}
|
|
}
|
|
# foreach my $parameter (keys %VALUE) {
|
|
# print "\t$parameter = $VALUE{$parameter}\n";
|
|
# }
|
|
return %VALUE;
|
|
}
|
|
|
|
sub check_if_crashed {
|
|
my ($i,$version,$no_wait) = @_;
|
|
$version = $VERSION unless $version; # default: current version
|
|
my $file = &versionize(&step_file($i),$version).".STDERR";
|
|
|
|
# while running, sometimes the STDERR file is slow in appearing - wait a bit just in case
|
|
if ($version == $VERSION && !$no_wait) {
|
|
my $j = 0;
|
|
while (! -e $file && $j < 100) {
|
|
sleep(5);
|
|
$j++;
|
|
}
|
|
}
|
|
|
|
#print "checking if $DO_STEP[$i]($version) crashed -> $file...\n";
|
|
return 1 if ! -e $file;
|
|
|
|
# check digest file (if it exists)
|
|
if (-e $file.".digest") {
|
|
my $error = 0;
|
|
open(DIGEST,$file.".digest") or die "Cannot open: $!";
|
|
while(<DIGEST>) {
|
|
print "\t$DO_STEP[$i]($version) crashed: $_" if $VERBOSE;
|
|
$error++;
|
|
}
|
|
close(DIGEST);
|
|
return $error;
|
|
}
|
|
|
|
# check against specified error patterns
|
|
my @DIGEST;
|
|
open(ERROR,$file) or die "Cannot open: $!";
|
|
while(<ERROR>) {
|
|
foreach my $pattern (@{$ERROR{&defined_step_id($i)}},
|
|
'error','killed','core dumped','can\'t read',
|
|
'no such file or directory','unknown option',
|
|
'died at','exit code','permission denied',
|
|
'segmentation fault','abort',
|
|
'no space left on device', ': not found',
|
|
'can\'t locate', 'unrecognized option', 'Exception') {
|
|
if (/$pattern/i) {
|
|
my $not_error = 0;
|
|
if (defined($NOT_ERROR{&defined_step_id($i)})) {
|
|
foreach my $override (@{$NOT_ERROR{&defined_step_id($i)}}) {
|
|
$not_error++ if /$override/i;
|
|
}
|
|
}
|
|
if (!$not_error) {
|
|
push @DIGEST,$pattern;
|
|
print "\t$DO_STEP[$i]($version) crashed: $pattern\n" if $VERBOSE;
|
|
}
|
|
}
|
|
}
|
|
last if scalar(@DIGEST)>10
|
|
}
|
|
close(ERROR);
|
|
|
|
# check if output file empty
|
|
my $output = &get_default_file(&deconstruct_name($DO_STEP[$i]));
|
|
# currently only works for single output file
|
|
if (-e $output && -z $output) {
|
|
push @DIGEST,"output file $output is empty";
|
|
}
|
|
|
|
# save digest file
|
|
open(DIGEST,">$file.digest") or die "Cannot open: $!";
|
|
foreach (@DIGEST) {
|
|
print DIGEST $_."\n";
|
|
}
|
|
close(DIGEST);
|
|
return scalar(@DIGEST);
|
|
}
|
|
|
|
# returns the name of the file where the step job is defined in
|
|
sub step_file {
|
|
my ($i) = @_;
|
|
my $step = $DO_STEP[$i];
|
|
$step =~ s/:/_/g;
|
|
my $dir = &check_and_get("GENERAL:working-dir");
|
|
return "$dir/steps/$step";
|
|
}
|
|
|
|
sub step_file2 {
|
|
my ($module,$set,$step) = @_;
|
|
my $dir = &check_and_get("GENERAL:working-dir");
|
|
`mkdir -p $dir/steps` if ! -e "$dir/steps";
|
|
my $file = "$dir/steps/$module" . ($set ? ("_".$set) : "") . "_$step";
|
|
return $file;
|
|
}
|
|
|
|
sub versionize {
|
|
my ($file,$version) = @_;
|
|
$version = $VERSION unless $version;
|
|
$file =~ s/steps\//steps\/$version\//;
|
|
return $file.".".$version;
|
|
}
|
|
|
|
sub defined_step_id {
|
|
my ($i) = @_;
|
|
return &defined_step($DO_STEP[$i]);
|
|
}
|
|
|
|
sub defined_step {
|
|
my ($step) = @_;
|
|
my $defined_step = $step;
|
|
$defined_step =~ s/:.+:/:/;
|
|
return $defined_step;
|
|
}
|
|
|
|
sub construct_name {
|
|
my ($module,$set,$step) = @_;
|
|
if (!defined($set) || $set eq "") {
|
|
return "$module:$step";
|
|
}
|
|
return "$module:$set:$step";
|
|
}
|
|
|
|
sub deconstruct_name {
|
|
my ($name) = @_;
|
|
my ($module,$set,$step);
|
|
if ($name !~ /:.+:/) {
|
|
($module,$step) = split(/:/,$name);
|
|
$set = "";
|
|
}
|
|
else {
|
|
($module,$set,$step) = split(/:/,$name);
|
|
}
|
|
# print "deconstruct_name $name -> ($module,$set,$step)\n";
|
|
return ($module,$set,$step);
|
|
}
|
|
|
|
sub deconstruct_local_name {
|
|
my ($module,$set,$name) = @_;
|
|
if ($name =~ /^(.+):(.+)$/) {
|
|
$module = $1;
|
|
$name = $2;
|
|
}
|
|
return ($module,$set,$name);
|
|
}
|
|
|
|
sub extend_local_name {
|
|
my ($module,$set,$name) = @_;
|
|
return &construct_name(&deconstruct_local_name($module,$set,$name));
|
|
}
|
|
|
|
### definition of steps
|
|
|
|
sub define_corpus_factorize {
|
|
my ($step_id) = @_;
|
|
my $scripts = &check_backoff_and_get("TUNING:moses-script-dir");
|
|
|
|
my ($output,$input) = &get_output_and_input($step_id);
|
|
my $input_extension = &check_backoff_and_get("TRAINING:input-extension");
|
|
my $output_extension = &check_backoff_and_get("TRAINING:output-extension");
|
|
|
|
my $dir = &check_and_get("GENERAL:working-dir");
|
|
my $temp_dir = &check_and_get("INPUT-FACTOR:temp-dir") . ".$VERSION";
|
|
my $cmd = "mkdir -p $temp_dir\n"
|
|
. &factorize_one_language("INPUT-FACTOR",
|
|
"$input.$input_extension",
|
|
"$output.$input_extension",
|
|
&check_backoff_and_get_array("TRAINING:input-factors"),
|
|
$step_id)
|
|
. &factorize_one_language("OUTPUT-FACTOR",
|
|
"$input.$output_extension",
|
|
"$output.$output_extension",
|
|
&check_backoff_and_get_array("TRAINING:output-factors"),
|
|
$step_id);
|
|
|
|
&create_step($step_id,$cmd);
|
|
}
|
|
|
|
sub define_tuningevaluation_factorize {
|
|
my ($step_id) = @_;
|
|
my $scripts = &check_backoff_and_get("TUNING:moses-script-dir");
|
|
|
|
my $dir = &check_and_get("GENERAL:working-dir");
|
|
my ($output,$input) = &get_output_and_input($step_id);
|
|
|
|
my $temp_dir = &check_and_get("INPUT-FACTOR:temp-dir") . ".$VERSION";
|
|
my $cmd = "mkdir -p $temp_dir\n"
|
|
. &factorize_one_language("INPUT-FACTOR",$input,$output,
|
|
&check_backoff_and_get_array("TRAINING:input-factors"),
|
|
$step_id);
|
|
|
|
&create_step($step_id,$cmd);
|
|
}
|
|
|
|
sub define_lm_factorize {
|
|
my ($step_id,$set) = @_;
|
|
my $scripts = &check_backoff_and_get("TUNING:moses-script-dir");
|
|
|
|
my ($output,$input) = &get_output_and_input($step_id);
|
|
print "LM:$set:factors\n" if $VERBOSE;
|
|
my $factor = &check_backoff_and_get_array("LM:$set:factors");
|
|
|
|
my $dir = &check_and_get("GENERAL:working-dir");
|
|
my $temp_dir = &check_and_get("INPUT-FACTOR:temp-dir") . ".$VERSION";
|
|
my $cmd = "mkdir -p $temp_dir\n"
|
|
. &factorize_one_language("OUTPUT-FACTOR",$input,$output,$factor,$step_id);
|
|
|
|
&create_step($step_id,$cmd);
|
|
}
|
|
|
|
sub define_interpolated_lm_factorize_tuning {
|
|
my ($step_id) = @_;
|
|
my $scripts = &check_backoff_and_get("TUNING:moses-script-dir");
|
|
|
|
my ($output,$input) = &get_output_and_input($step_id);
|
|
my $factor = &check_backoff_and_get_array("TRAINING:output-factors");
|
|
|
|
my $dir = &check_and_get("GENERAL:working-dir");
|
|
my $temp_dir = &check_and_get("INPUT-FACTOR:temp-dir") . ".$VERSION";
|
|
my $cmd = "mkdir -p $temp_dir\n"
|
|
. &factorize_one_language("OUTPUT-FACTOR",$input,$output,$factor,$step_id);
|
|
|
|
&create_step($step_id,$cmd);
|
|
}
|
|
|
|
sub define_splitter_train {
|
|
my ($step_id,$set) = @_;
|
|
|
|
my ($output,$input) = &get_output_and_input($step_id);
|
|
my $input_splitter = &get("GENERAL:input-splitter");
|
|
my $output_splitter = &get("GENERAL:output-splitter");
|
|
my $input_extension = &check_backoff_and_get("SPLITTER:input-extension");
|
|
my $output_extension = &check_backoff_and_get("SPLITTER:output-extension");
|
|
|
|
my $cmd = "";
|
|
if ($input_splitter) {
|
|
$cmd .= "$input_splitter -train -model $output.$input_extension -corpus $input.$input_extension\n";
|
|
}
|
|
if ($output_splitter) {
|
|
$cmd .= "$output_splitter -train -model $output.$output_extension -corpus $input.$output_extension\n";
|
|
}
|
|
|
|
&create_step($step_id,$cmd);
|
|
}
|
|
|
|
sub define_lm_train_randomized {
|
|
my ($step_id,$set) = @_;
|
|
my $training = &check_backoff_and_get("LM:$set:rlm-training");
|
|
my $order = &check_backoff_and_get("LM:$set:order");
|
|
my ($output,$input) = &get_output_and_input($step_id);
|
|
|
|
$output =~ /^(.+)\/([^\/]+)$/;
|
|
my ($output_dir,$output_prefix) = ($1,$2);
|
|
my $cmd = "gzip $input\n";
|
|
$cmd .= "$training -struct BloomMap -order $order -output-prefix $output_prefix -output-dir $output_dir -input-type corpus -input-path $input\n";
|
|
$cmd .= "gunzip $input\n";
|
|
$cmd .= "mv $output.BloomMap $output\n";
|
|
|
|
&create_step($step_id,$cmd);
|
|
}
|
|
|
|
sub define_lm_train_bilingual_lm {
|
|
my ($step_id,$set) = @_;
|
|
my ($working_dir, $ngrams, $corpus) = &get_output_and_input($step_id);
|
|
my $scripts = &check_backoff_and_get("LM:moses-script-dir");
|
|
my $cmd = "$scripts/training/bilingual-lm/train_nplm.py -w $working_dir -c $corpus -r $working_dir";
|
|
my $nplm_dir = &check_backoff_and_get("LM:$set:nplm-dir");
|
|
$cmd .= " -l $nplm_dir";
|
|
|
|
my ($n, $m, $total_order) = &get_bilingual_lm_order($set);
|
|
$cmd .= " -n $total_order";
|
|
|
|
my $epochs = &get_bilingual_lm_epochs($set);
|
|
$cmd .= " -e $epochs" if defined($epochs);
|
|
|
|
my $nplm_settings = backoff_and_get("LM:$set:nplm-settings");
|
|
$cmd .= " --extra-settings \"$nplm_settings\"" if defined($nplm_settings);
|
|
|
|
# Create the ini file
|
|
$cmd .= "\n";
|
|
$cmd .= "$scripts/training/bilingual-lm/create_blm_ini.py -w $working_dir -n $n -m $m -x $set -e $epochs";
|
|
|
|
&create_step($step_id,$cmd);
|
|
}
|
|
|
|
sub define_lm_prepare_bilingual_lm {
|
|
my ($step_id,$set) = @_;
|
|
my ($working_dir, $corpus, $align) = &get_output_and_input($step_id);
|
|
my $scripts = &check_backoff_and_get("LM:moses-script-dir");
|
|
my $cmd = "$scripts/training/bilingual-lm/extract_training.py -w $working_dir -c $corpus";
|
|
|
|
my $input_extension = &check_backoff_and_get("GENERAL:input-extension");
|
|
my $output_extension = &check_backoff_and_get("GENERAL:output-extension");
|
|
$cmd .= " -e $output_extension -f $input_extension";
|
|
|
|
my $align_method = &check_backoff_and_get("TRAINING:alignment-symmetrization-method");
|
|
$cmd .= " -a $align.$align_method";
|
|
|
|
my ($n, $m, $total_order) = &get_bilingual_lm_order($set);
|
|
$cmd .= " -n $n -m $m";
|
|
|
|
my $bilingual_settings = backoff_and_get("LM:$set:bilingual-lm-settings");
|
|
$cmd .= " $bilingual_settings" if defined($bilingual_settings);
|
|
|
|
|
|
&create_step($step_id,$cmd);
|
|
}
|
|
|
|
sub define_lm_train_nplm {
|
|
my ($step_id,$set) = @_;
|
|
my ($working_dir, $corpus) = &get_output_and_input($step_id);
|
|
my $scripts = &check_backoff_and_get("LM:moses-script-dir");
|
|
my $cmd = "$scripts/training/train-neurallm.py --mmap --working-dir $working_dir --corpus $corpus";
|
|
my $nplm_dir = &check_backoff_and_get("LM:$set:nplm-dir");
|
|
$cmd .= " --nplm-home $nplm_dir";
|
|
|
|
my $epochs = &backoff_and_get("LM:$set:epochs");
|
|
$epochs = 2 unless defined($epochs);
|
|
$cmd .= " --epochs $epochs";
|
|
|
|
my $nplm_settings = backoff_and_get("LM:$set:nplm-settings");
|
|
$cmd .= " $nplm_settings" if defined($nplm_settings);
|
|
|
|
my $order = &backoff_and_get("LM:$set:order");
|
|
$order = 5 unless defined($order);
|
|
$cmd .= " --order $order";
|
|
|
|
# Create the ini file
|
|
$cmd .= "\n";
|
|
$cmd .= "$scripts/training/create_nplm_ini.py -w $working_dir -e $epochs -x $set -n $order";
|
|
|
|
&create_step($step_id,$cmd);
|
|
}
|
|
|
|
sub get_bilingual_lm_order {
|
|
my ($set) = @_;
|
|
my $order = &backoff_and_get("LM:$set:order");
|
|
$order = 5 unless defined ($order);
|
|
my $source_window = &backoff_and_get("LM:$set:source-window");
|
|
$source_window = 4 unless defined ($order);
|
|
return ($order, $source_window, $order + 2*$source_window+1);
|
|
}
|
|
|
|
sub get_bilingual_lm_epochs {
|
|
my ($set) = @_;
|
|
my $epochs = &backoff_and_get("LM:$set:epochs");
|
|
$epochs = 10 unless defined($epochs);
|
|
return $epochs;
|
|
}
|
|
|
|
sub define_lm_randomize {
|
|
my ($step_id,$set_dummy) = @_;
|
|
|
|
my ($module,$set,$stepname) = &deconstruct_name($DO_STEP[$step_id]);
|
|
my $randomizer = &check_backoff_and_get("$module:$set:lm-randomizer");
|
|
my $order = &check_backoff_and_get("$module:$set:order");
|
|
my ($output,$input) = &get_output_and_input($step_id);
|
|
|
|
$output =~ /^(.+)\/([^\/]+)$/;
|
|
my ($output_dir,$output_prefix) = ($1,$2);
|
|
my $cmd = "$randomizer -struct BloomMap -order $order -output-prefix $output_prefix -output-dir $output_dir -input-type arpa -input-path $input\n";
|
|
$cmd .= "mv $output.BloomMap $output\n";
|
|
|
|
&create_step($step_id,$cmd);
|
|
}
|
|
|
|
sub factorize_one_language {
|
|
my ($type,$infile,$outfile,$FACTOR,$step_id) = @_;
|
|
my $scripts = &check_backoff_and_get("TUNING:moses-script-dir");
|
|
my $temp_dir = &check_and_get("INPUT-FACTOR:temp-dir") . ".$VERSION";
|
|
my $parallelizer = &get("GENERAL:generic-parallelizer");
|
|
my ($module,$set,$stepname) = &deconstruct_name($DO_STEP[$step_id]);
|
|
|
|
my ($cmd,$list) = ("");
|
|
foreach my $factor (@{$FACTOR}) {
|
|
if ($factor eq "word") {
|
|
$list .= " $infile";
|
|
}
|
|
else {
|
|
my $script = &check_and_get("$type:$factor:factor-script");
|
|
my $out = "$outfile.$factor";
|
|
if ($parallelizer && defined($PARALLELIZE{&defined_step($DO_STEP[$step_id])})
|
|
&& ( (&get("$module:jobs") && $CLUSTER)
|
|
|| (&get("$module:cores") && $MULTICORE))) {
|
|
my $subdir = $module;
|
|
$subdir =~ tr/A-Z/a-z/;
|
|
$subdir .= "/tmp.$set.$stepname.$type.$factor.$VERSION";
|
|
if ($CLUSTER) {
|
|
my $qflags = "";
|
|
my $qsub_args = &get_qsub_args($DO_STEP[$step_id]);
|
|
$qflags="--queue-flags \"$qsub_args\"" if ($CLUSTER && $qsub_args);
|
|
$cmd .= "$parallelizer $qflags -in $infile -out $out -cmd '$script %s %s $temp_dir/$subdir' -jobs ".&get("$module:jobs")." -tmpdir $temp_dir/$subdir\n";
|
|
$QSUB_STEP{$step_id}++;
|
|
}
|
|
elsif ($MULTICORE) {
|
|
$cmd .= "$parallelizer -in $infile -out $out -cmd '$script %s %s $temp_dir/$subdir' -cores ".&get("$module:cores")." -tmpdir $temp_dir/$subdir\n";
|
|
}
|
|
}
|
|
else {
|
|
$cmd .= "$script $infile $out $temp_dir\n";
|
|
}
|
|
$list .= " $out";
|
|
}
|
|
}
|
|
return $cmd . "$scripts/training/combine_factors.pl $list > $outfile\n";
|
|
}
|
|
|
|
sub define_tuning_tune {
|
|
my ($step_id) = @_;
|
|
my $dir = &check_and_get("GENERAL:working-dir");
|
|
my $hierarchical = &get("TRAINING:hierarchical-rule-set");
|
|
my $tuning_script = &check_and_get("TUNING:tuning-script");
|
|
my $use_mira = &backoff_and_get("TUNING:use-mira", 0);
|
|
my $word_alignment = &backoff_and_get("TRAINING:include-word-alignment-in-rules");
|
|
my $tmp_dir = &get_tmp_file("TUNING","","tune");
|
|
|
|
# the last 3 variables are only used for mira tuning
|
|
my ($tuned_config,$config,$input,$reference,$config_devtest,$input_devtest,$reference_devtest, $filtered_config) = &get_output_and_input($step_id);
|
|
$config = $filtered_config if $filtered_config;
|
|
|
|
|
|
my $cmd = "";
|
|
if ($use_mira) {
|
|
my $addTags = &backoff_and_get("TUNING:add-tags");
|
|
my $use_jackknife = &backoff_and_get("TUNING:use-jackknife");
|
|
if ($addTags && !$use_jackknife) {
|
|
my $input_with_tags = $input.".".$VERSION.".tags";
|
|
`$addTags < $input > $input_with_tags`;
|
|
$input = $input_with_tags;
|
|
}
|
|
|
|
my $addTagsDevtest = &backoff_and_get("TUNING:add-tags-devtest");
|
|
if ($addTagsDevtest) {
|
|
my $input_devtest_with_tags = $input_devtest.".".$VERSION.".tags";
|
|
`$addTagsDevtest < $input_devtest > $input_devtest_with_tags`;
|
|
$input_devtest = $input_devtest_with_tags;
|
|
}
|
|
|
|
system("mkdir -p $tmp_dir");
|
|
|
|
my $mira_config = "$tmp_dir/mira-config.$VERSION.";
|
|
my $mira_config_log = $mira_config."log";
|
|
$mira_config .= "cfg";
|
|
|
|
write_mira_config($mira_config,$tmp_dir,$config,$input,$reference,$config_devtest,$input_devtest,$reference_devtest);
|
|
#$cmd = "$tuning_script -config $mira_config -exec >& $mira_config_log";
|
|
# we want error messages in top-level log file
|
|
$cmd = "$tuning_script -config $mira_config -exec ";
|
|
|
|
# write script to select the best set of weights after training for the specified number of epochs -->
|
|
# cp to tuning/tmp.?/moses.ini
|
|
my $script_filename = "$tmp_dir/selectBestWeights.";
|
|
my $script_filename_log = $script_filename."log";
|
|
$script_filename .= "perl";
|
|
my $weight_output_file = "$tmp_dir/moses.ini";
|
|
write_selectBestMiraWeights($tmp_dir, $script_filename, $weight_output_file);
|
|
$cmd .= "\n$script_filename >& $script_filename_log";
|
|
}
|
|
else {
|
|
my $scripts = &check_backoff_and_get("TUNING:moses-script-dir");
|
|
my $nbest_size = &check_and_get("TUNING:nbest");
|
|
my $lambda = &backoff_and_get("TUNING:lambda");
|
|
my $tune_continue = &backoff_and_get("TUNING:continue");
|
|
my $skip_decoder = &backoff_and_get("TUNING:skip-decoder");
|
|
my $tune_inputtype = &backoff_and_get("TUNING:inputtype");
|
|
my $jobs = &backoff_and_get("TUNING:jobs");
|
|
my $decoder = &check_backoff_and_get("TUNING:decoder");
|
|
my $cache_model = &backoff_and_get("GENERAL:cache-model");
|
|
|
|
if (defined($cache_model) && !($jobs && $jobs>1 && $CLUSTER)) {
|
|
$cmd .= "MOSES_INI=`$scripts/ems/support/cache-model.perl $config $cache_model`\n";
|
|
$config = "\$MOSES_INI";
|
|
}
|
|
|
|
my $decoder_settings = &backoff_and_get("TUNING:decoder-settings");
|
|
$decoder_settings = "" unless $decoder_settings;
|
|
$decoder_settings .= " -v 0 " unless $CLUSTER && $jobs && $jobs>1;
|
|
|
|
my $tuning_settings = &backoff_and_get("TUNING:tuning-settings");
|
|
$tuning_settings = "" unless $tuning_settings;
|
|
|
|
$cmd .= "$tuning_script $input $reference $decoder $config --nbest $nbest_size --working-dir $tmp_dir --decoder-flags \"$decoder_settings\" --rootdir $scripts $tuning_settings --no-filter-phrase-table";
|
|
$cmd .= " --lambdas \"$lambda\"" if $lambda;
|
|
$cmd .= " --continue" if $tune_continue;
|
|
$cmd .= " --skip-decoder" if $skip_decoder;
|
|
$cmd .= " --inputtype $tune_inputtype" if defined($tune_inputtype);
|
|
|
|
my $qsub_args = &get_qsub_args($DO_STEP[$step_id]);
|
|
$cmd .= " --queue-flags=\"$qsub_args\"" if ($CLUSTER && $qsub_args);
|
|
$cmd .= " --jobs $jobs" if $CLUSTER && $jobs && $jobs>1;
|
|
$cmd .= " --cache-model $cache_model" if $cache_model && $CLUSTER && $jobs && $jobs>1;
|
|
my $tuning_dir = $tuned_config;
|
|
$tuning_dir =~ s/\/[^\/]+$//;
|
|
$cmd .= "\nmkdir -p $tuning_dir";
|
|
}
|
|
|
|
$cmd .= "\ncp $tmp_dir/moses.ini $tuned_config";
|
|
|
|
&create_step($step_id,$cmd);
|
|
}
|
|
|
|
sub write_mira_config {
|
|
my ($config_filename,$expt_dir,$tune_filtered_ini,$input,$reference,$devtest_filtered_ini,$input_devtest,$reference_devtest) = @_;
|
|
my $moses_src_dir = &check_and_get("GENERAL:moses-src-dir");
|
|
my $mira_src_dir = &backoff_and_get("GENERAL:mira-src-dir");
|
|
my $tuning_decoder_settings = &check_and_get("TUNING:decoder-settings");
|
|
my $start_weights = &backoff_and_get("TUNING:start-weight-config");
|
|
my $tuning_settings = &check_and_get("TUNING:tuning-settings");
|
|
|
|
my $parallel_settings = &backoff_and_get("TUNING:parallel-settings");
|
|
my $use_jackknife = &backoff_and_get("TUNING:use-jackknife");
|
|
|
|
# are we tuning a meta feature?
|
|
my $tune_meta_feature = &backoff_and_get("TUNING:tune-meta-feature");
|
|
|
|
my $tune_filtered_ini_start;
|
|
if (!$use_jackknife) {
|
|
$tune_filtered_ini =~ /.*\/([A-Za-z0-9\.\-\_]*)$/;
|
|
$tune_filtered_ini_start = $1;
|
|
$tune_filtered_ini_start = $expt_dir."/".$tune_filtered_ini_start.".start";
|
|
if ($start_weights) {
|
|
# apply start weights to filtered ini file, and pass the new ini to mira
|
|
print "DEBUG: $RealBin/support/substitute-weights.perl $start_weights $tune_filtered_ini $tune_filtered_ini_start \n";
|
|
system("$RealBin/support/substitute-weights.perl $start_weights $tune_filtered_ini $tune_filtered_ini_start");
|
|
}
|
|
}
|
|
|
|
# do we want to continue an interrupted experiment?
|
|
my $continue_expt = &backoff_and_get("TUNING:continue-expt");
|
|
my $continue_epoch = &backoff_and_get("TUNING:continue-epoch");
|
|
my $continue_weights = &backoff_and_get("TUNING:continue-weights");
|
|
|
|
# mira config file
|
|
open(CFG, ">$config_filename");
|
|
print CFG "[general] \n";
|
|
print CFG "name=expt \n";
|
|
print CFG "fold=0 \n";
|
|
print CFG "mpienv=openmpi_fillup_mark2 \n";
|
|
if ($mira_src_dir) {
|
|
print CFG "moses-home=".$mira_src_dir."\n";
|
|
}
|
|
else {
|
|
print CFG "moses-home=".$moses_src_dir."\n";
|
|
}
|
|
print CFG "working-dir=".$expt_dir."\n";
|
|
if ($continue_expt && $continue_expt > 0) {
|
|
print CFG "continue-expt=".$continue_expt."\n";
|
|
print CFG "continue-epoch=".$continue_epoch."\n";
|
|
print CFG "continue-weights=".$continue_weights."\n";
|
|
}
|
|
print CFG "tune-meta-feature=1 \n" if ($tune_meta_feature);
|
|
print CFG "jackknife=1 \n" if ($use_jackknife);
|
|
print CFG "wait-for-bleu=1 \n\n";
|
|
#print CFG "decoder-settings=".$tuning_decoder_settings."\n\n";
|
|
print CFG "[train] \n";
|
|
print CFG "trainer=\${moses-home}/bin/mira \n";
|
|
if ($use_jackknife) {
|
|
print CFG "input-files-folds=";
|
|
for my $i (0..9) {
|
|
my $addTags = &backoff_and_get("TUNING:add-tags");
|
|
if ($addTags) {
|
|
my $input_with_tags = $input.".".$VERSION.".tags";
|
|
`$addTags.only$i < $input.only$i > $input_with_tags.only$i`;
|
|
|
|
print CFG $input_with_tags.".only$i, " if $i<9;
|
|
print CFG $input_with_tags.".only$i" if $i==9;
|
|
}
|
|
else {
|
|
print CFG $input.".only$i, " if $i<9;
|
|
print CFG $input.".only$i" if $i==9;
|
|
}
|
|
}
|
|
print CFG "\n";
|
|
print CFG "reference-files-folds=";
|
|
for my $i (0..9) {
|
|
print CFG $reference.".only$i, " if $i<9;
|
|
print CFG $reference.".only$i" if $i==9;
|
|
}
|
|
print CFG "\n";
|
|
print CFG "moses-ini-files-folds=";
|
|
for my $i (0..9) {
|
|
print CFG $start_weights.".wo$i, " if $i<9;
|
|
print CFG $start_weights.".wo$i" if $i==9;
|
|
}
|
|
print CFG "\n";
|
|
}
|
|
else {
|
|
print CFG "input-file=".$input."\n";
|
|
print CFG "reference-files=".$reference."\n";
|
|
if ($start_weights) {
|
|
print CFG "moses-ini-file=".$tune_filtered_ini_start."\n";
|
|
}
|
|
else {
|
|
print CFG "moses-ini-file=".$tune_filtered_ini."\n";
|
|
}
|
|
}
|
|
print CFG "decoder-settings=".$tuning_decoder_settings." -text-type \"dev\"\n";
|
|
print CFG "hours=48 \n";
|
|
if ($parallel_settings) {
|
|
foreach my $setting (split(" ", $parallel_settings)) {
|
|
print CFG $setting."\n";
|
|
}
|
|
}
|
|
print CFG "extra-args=".$tuning_settings."\n\n";
|
|
print CFG "[devtest] \n";
|
|
if (&get("TRAINING:hierarchical-rule-set")) {
|
|
print CFG "moses=\${moses-home}/bin/moses_chart \n";
|
|
}
|
|
else {
|
|
print CFG "moses=\${moses-home}/bin/moses \n";
|
|
}
|
|
# use multi-bleu to select the best set of weights
|
|
print CFG "bleu=\${moses-home}/scripts/generic/multi-bleu.perl \n";
|
|
print CFG "input-file=".$input_devtest."\n";
|
|
print CFG "reference-file=".$reference_devtest."\n";
|
|
print CFG "moses-ini-file=".$devtest_filtered_ini."\n";
|
|
print CFG "decoder-settings=".$tuning_decoder_settings." -text-type \"devtest\"\n";
|
|
print CFG "hours=12 \nextra-args= \nskip-dev=1 \nskip-devtest=0 \nskip-submit=0 \n";
|
|
close(CFG);
|
|
}
|
|
|
|
sub write_selectBestMiraWeights {
|
|
my ($expt_dir, $script_filename, $weight_out_file) = @_;
|
|
open(SCR, ">$script_filename");
|
|
|
|
print SCR "#!/usr/bin/perl -w \nuse strict; \n\n";
|
|
print SCR "my \@devtest_bleu = glob(\"$expt_dir/*_devtest.bleu\"); \# expt_00_0_devtest.bleu \n";
|
|
print SCR "if (scalar(\@devtest_bleu) == 0) { \n";
|
|
print SCR "\tprint STDERR \"ERROR: no bleu files globbed, cannot find best weights.\\n\"; \n";
|
|
print SCR "\texit(1); \n";
|
|
print SCR "} \n\n";
|
|
print SCR "my (\$best_weights, \$best_id); \n";
|
|
print SCR "my \$best_bleu = -1; \n";
|
|
print SCR "my \$best_ratio = 0; \n";
|
|
print SCR "foreach my \$bleu_file (\@devtest_bleu) { \n";
|
|
print SCR "\t\$bleu_file =~ /_([\\d_]+)_devtest.bleu/; \n";
|
|
print SCR "\tmy \$id = \$1; \n";
|
|
print SCR "\topen(BLEU, \$bleu_file); \n";
|
|
print SCR "\tmy \$bleu = <BLEU>; \n";
|
|
print SCR "\t\$bleu =~ /BLEU = ([\\d\\.]+), .*ratio=([\\d\\.]+), /; \n";
|
|
print SCR "\tif (\$1 > \$best_bleu || (\$1 == \$best_bleu && (abs(1-\$2) < abs(1-\$best_ratio)))) { \n";
|
|
print SCR "\t\t\$best_bleu = \$1; \n";
|
|
print SCR "\t\t\$best_ratio = \$2; \n";
|
|
print SCR "\t\t# expt1-devtest.00_0.ini (incl. path to sparse weights) \n";
|
|
print SCR "\t\t(\$best_weights) = glob(\"$expt_dir/*devtest.\$id.ini\"); \n";
|
|
print SCR "\t} \n";
|
|
print SCR "} \n\n";
|
|
print SCR "print STDERR \"Best weights according to BLEU on devtest set: \$best_weights \\n\"; \n";
|
|
print SCR "system(\"cp \$best_weights $weight_out_file\"); \n\n";
|
|
|
|
close(SCR);
|
|
system("chmod u+x $script_filename");
|
|
}
|
|
|
|
sub define_training_prepare_data_fast_align {
|
|
my ($step_id) = @_;
|
|
|
|
my ($prepared, $corpus) = &get_output_and_input($step_id);
|
|
my $scripts = &check_and_get("GENERAL:moses-script-dir");
|
|
my $input_extension = &check_backoff_and_get("TRAINING:input-extension");
|
|
my $output_extension = &check_backoff_and_get("TRAINING:output-extension");
|
|
|
|
my $alignment_factors = "";
|
|
if (&backoff_and_get("TRAINING:input-factors")) {
|
|
my %IN = &get_factor_id("input");
|
|
my %OUT = &get_factor_id("output");
|
|
$alignment_factors = &encode_factor_definition("alignment-factors",\%IN,\%OUT);
|
|
}
|
|
my $cmd = "$scripts/ems/support/prepare-fast-align.perl $corpus.$input_extension $corpus.$output_extension $alignment_factors > $prepared";
|
|
|
|
&create_step($step_id,$cmd);
|
|
}
|
|
|
|
sub define_training_prepare_data {
|
|
my ($step_id) = @_;
|
|
|
|
my ($prepared, $corpus) = &get_output_and_input($step_id);
|
|
my $cmd = &get_training_setting(1);
|
|
$cmd .= "-corpus $corpus ";
|
|
$cmd .= "-corpus-dir $prepared ";
|
|
|
|
&create_step($step_id,$cmd);
|
|
}
|
|
|
|
sub define_training_run_giza {
|
|
my ($step_id) = @_;
|
|
|
|
my ($giza, $prepared) = &get_output_and_input($step_id);
|
|
my $cmd = &get_training_setting(2);
|
|
$cmd .= "-corpus-dir $prepared ";
|
|
$cmd .= "-giza-e2f $giza ";
|
|
$cmd .= "-direction 2 ";
|
|
|
|
&create_step($step_id,$cmd);
|
|
}
|
|
|
|
sub define_training_run_giza_inverse {
|
|
my ($step_id) = @_;
|
|
|
|
my ($giza, $prepared) = &get_output_and_input($step_id);
|
|
my $cmd = &get_training_setting(2);
|
|
$cmd .= "-corpus-dir $prepared ";
|
|
$cmd .= "-giza-f2e $giza ";
|
|
$cmd .= "-direction 1 ";
|
|
|
|
&create_step($step_id,$cmd);
|
|
}
|
|
|
|
sub define_training_symmetrize_giza {
|
|
my ($step_id) = @_;
|
|
|
|
my ($aligned, $giza,$giza_inv) = &get_output_and_input($step_id);
|
|
my $method = &check_and_get("TRAINING:alignment-symmetrization-method");
|
|
my $cmd = &get_training_setting(3);
|
|
my $alignment_stem = &versionize(&long_file_name("aligned","model",""));
|
|
|
|
$cmd .= "-giza-e2f $giza -giza-f2e $giza_inv ";
|
|
$cmd .= "-alignment-file $aligned ";
|
|
$cmd .= "-alignment-stem $alignment_stem ";
|
|
$cmd .= "-alignment $method ";
|
|
|
|
&create_step($step_id,$cmd);
|
|
}
|
|
|
|
sub define_training_build_suffix_array {
|
|
my ($step_id) = @_;
|
|
|
|
my $scripts = &check_and_get("GENERAL:moses-script-dir");
|
|
|
|
my ($model, $aligned,$corpus) = &get_output_and_input($step_id);
|
|
my $sa_exec_dir = &check_and_get("TRAINING:suffix-array");
|
|
my $input_extension = &check_backoff_and_get("TRAINING:input-extension");
|
|
my $output_extension = &check_backoff_and_get("TRAINING:output-extension");
|
|
my $method = &check_and_get("TRAINING:alignment-symmetrization-method");
|
|
|
|
my $glue_grammar_file = &versionize(&long_file_name("glue-grammar","model",""));
|
|
|
|
my $cmd = "$scripts/training/wrappers/adam-suffix-array/suffix-array-create.sh $sa_exec_dir $corpus.$input_extension $corpus.$output_extension $aligned.$method $model $glue_grammar_file";
|
|
|
|
&create_step($step_id,$cmd);
|
|
}
|
|
|
|
sub define_training_build_biconcor {
|
|
my ($step_id) = @_;
|
|
|
|
my ($model, $aligned,$corpus) = &get_output_and_input($step_id);
|
|
my $biconcor = &check_and_get("TRAINING:biconcor");
|
|
my $input_extension = &check_backoff_and_get("TRAINING:input-extension");
|
|
my $output_extension = &check_backoff_and_get("TRAINING:output-extension");
|
|
my $method = &check_and_get("TRAINING:alignment-symmetrization-method");
|
|
|
|
my $cmd = "$biconcor -c $corpus.$input_extension -t $corpus.$output_extension -a $aligned.$method -s $model";
|
|
&create_step($step_id,$cmd);
|
|
}
|
|
|
|
sub define_training_build_lex_trans {
|
|
my ($step_id) = @_;
|
|
|
|
my ($lex, $aligned,$corpus) = &get_output_and_input($step_id);
|
|
my $baseline_alignment = &get("TRAINING:baseline-alignment");
|
|
my $baseline_corpus = &get("TRAINING:baseline-corpus");
|
|
my $alignment_stem = &versionize(&long_file_name("aligned","model",""));
|
|
$alignment_stem = $CONFIG{"TRAINING:word-alignment"}[0] if defined($CONFIG{"TRAINING:word-alignment"});
|
|
|
|
my $cmd = &get_training_setting(4);
|
|
$cmd .= "-lexical-file $lex ";
|
|
$cmd .= "-alignment-file $aligned ";
|
|
$cmd .= "-alignment-stem $alignment_stem ";
|
|
$cmd .= "-corpus $corpus ";
|
|
$cmd .= "-baseline-corpus $baseline_corpus " if defined($baseline_corpus) && defined($baseline_alignment);
|
|
$cmd .= "-baseline-alignment $baseline_alignment " if defined($baseline_corpus) && defined($baseline_alignment);
|
|
|
|
&create_step($step_id,$cmd);
|
|
}
|
|
|
|
sub define_training_build_transliteration_model {
|
|
my ($step_id) = @_;
|
|
|
|
my ($model, $corpus, $alignment) = &get_output_and_input($step_id);
|
|
|
|
my $moses_script_dir = &check_and_get("GENERAL:moses-script-dir");
|
|
my $input_extension = &check_backoff_and_get("TRAINING:input-extension");
|
|
my $output_extension = &check_backoff_and_get("TRAINING:output-extension");
|
|
my $sym_method = &check_and_get("TRAINING:alignment-symmetrization-method");
|
|
my $moses_src_dir = &check_and_get("GENERAL:moses-src-dir");
|
|
my $external_bin_dir = &check_and_get("GENERAL:external-bin-dir");
|
|
my $srilm_dir = &check_and_get("TRAINING:srilm-dir");
|
|
my $decoder = &get("TRAINING:transliteration-decoder");
|
|
|
|
my $cmd = "$moses_script_dir/Transliteration/train-transliteration-module.pl";
|
|
$cmd .= " --corpus-f $corpus.$input_extension";
|
|
$cmd .= " --corpus-e $corpus.$output_extension";
|
|
$cmd .= " --alignment $alignment.$sym_method";
|
|
$cmd .= " --out-dir $model";
|
|
$cmd .= " --moses-src-dir $moses_src_dir";
|
|
$cmd .= " --decoder $decoder" if defined($decoder);
|
|
$cmd .= " --external-bin-dir $external_bin_dir";
|
|
$cmd .= " --srilm-dir $srilm_dir";
|
|
$cmd .= " --input-extension $input_extension";
|
|
$cmd .= " --output-extension $output_extension";
|
|
$cmd .= " --factor 0-0";
|
|
$cmd .= " --source-syntax " if &get("GENERAL:input-parser");
|
|
$cmd .= " --target-syntax " if &get("GENERAL:output-parser");
|
|
|
|
&create_step($step_id, $cmd);
|
|
}
|
|
|
|
sub define_training_extract_phrases {
|
|
my ($step_id) = @_;
|
|
|
|
my ($extract, $aligned,$corpus) = &get_output_and_input($step_id);
|
|
my $cmd = &get_training_setting(5);
|
|
my $alignment_stem = &versionize(&long_file_name("aligned","model",""));
|
|
$alignment_stem = $CONFIG{"TRAINING:word-alignment"}[0] if defined($CONFIG{"TRAINING:word-alignment"});
|
|
|
|
$cmd .= "-alignment-file $aligned ";
|
|
$cmd .= "-alignment-stem $alignment_stem ";
|
|
$cmd .= "-extract-file $extract ";
|
|
$cmd .= "-corpus $corpus ";
|
|
|
|
if (&get("TRAINING:hierarchical-rule-set")) {
|
|
my $no_glue_grammar = &get("TRAINING:no-glue-grammar");
|
|
if (!defined($no_glue_grammar) || $no_glue_grammar eq "false") {
|
|
my $glue_grammar_file = &get("TRAINING:glue-grammar");
|
|
$glue_grammar_file = &versionize(&long_file_name("glue-grammar","model",""))
|
|
unless $glue_grammar_file;
|
|
$cmd .= "-glue-grammar-file $glue_grammar_file ";
|
|
}
|
|
|
|
if (&get("GENERAL:output-parser") && (&get("TRAINING:use-unknown-word-labels") || &get("TRAINING:use-unknown-word-soft-matches"))) {
|
|
my $unknown_word_label = &versionize(&long_file_name("unknown-word-label","model",""));
|
|
$cmd .= "-unknown-word-label $unknown_word_label ";
|
|
}
|
|
|
|
if (&get("GENERAL:output-parser") && &get("TRAINING:use-unknown-word-soft-matches")) {
|
|
my $unknown_word_soft_matches = &versionize(&long_file_name("unknown-word-soft-matches","model",""));
|
|
$cmd .= "-unknown-word-soft-matches $unknown_word_soft_matches ";
|
|
}
|
|
|
|
if (&get("TRAINING:use-ghkm")) {
|
|
$cmd .= "-ghkm ";
|
|
}
|
|
|
|
if (&get("TRAINING:ghkm-tree-fragments")) {
|
|
$cmd .= "-ghkm-tree-fragments ";
|
|
}
|
|
|
|
if (&get("TRAINING:ghkm-phrase-orientation")) {
|
|
$cmd .= "-ghkm-phrase-orientation ";
|
|
my $phrase_orientation_priors_file = &versionize(&long_file_name("phrase-orientation-priors","model",""));
|
|
$cmd .= "-phrase-orientation-priors-file $phrase_orientation_priors_file ";
|
|
}
|
|
|
|
if (&get("TRAINING:ghkm-source-labels")) {
|
|
$cmd .= "-ghkm-source-labels ";
|
|
my $source_labels_file = &versionize(&long_file_name("source-labels","model",""));
|
|
$cmd .= "-ghkm-source-labels-file $source_labels_file ";
|
|
}
|
|
|
|
if (&get("TRAINING:ghkm-parts-of-speech")) {
|
|
$cmd .= "-ghkm-parts-of-speech ";
|
|
my $parts_of_speech_labels_file = &versionize(&long_file_name("parts-of-speech","model",""));
|
|
$cmd .= "-ghkm-parts-of-speech-file $parts_of_speech_labels_file ";
|
|
}
|
|
|
|
if (&get("TRAINING:ghkm-parts-of-speech-factor")) {
|
|
$cmd .= "-ghkm-parts-of-speech-factor ";
|
|
}
|
|
|
|
if (&get("TRAINING:ghkm-strip-bitpar-nonterminal-labels")) {
|
|
$cmd .= "-ghkm-strip-bitpar-nonterminal-labels ";
|
|
}
|
|
}
|
|
|
|
my $extract_settings = &get("TRAINING:extract-settings");
|
|
$extract_settings .= " --IncludeSentenceId " if &get("TRAINING:domain-features");
|
|
$cmd .= "-extract-options '".$extract_settings."' " if defined($extract_settings);
|
|
|
|
my $baseline_extract = &get("TRAINING:baseline-extract");
|
|
$cmd .= "-baseline-extract $baseline_extract" if defined($baseline_extract);
|
|
|
|
&create_step($step_id,$cmd);
|
|
}
|
|
|
|
sub define_training_build_ttable {
|
|
my ($step_id) = @_;
|
|
|
|
my ($phrase_table, $extract,$lex,$domains) = &get_output_and_input($step_id);
|
|
my $word_report = &backoff_and_get("EVALUATION:report-precision-by-coverage");
|
|
my $word_alignment = &backoff_and_get("TRAINING:include-word-alignment-in-rules");
|
|
|
|
my $cmd = &get_training_setting(6);
|
|
$cmd .= "-extract-file $extract ";
|
|
$cmd .= "-lexical-file $lex ";
|
|
$cmd .= &get_table_name_settings("translation-factors","phrase-translation-table",$phrase_table);
|
|
|
|
$cmd .= "-no-word-alignment " if defined($word_alignment) && $word_alignment eq "no";
|
|
|
|
$cmd .= &define_domain_feature_score_option($domains) if &get("TRAINING:domain-features");
|
|
|
|
if (&get("TRAINING:hierarchical-rule-set")) {
|
|
|
|
if (&get("TRAINING:ghkm-tree-fragments")) {
|
|
$cmd .= "-ghkm-tree-fragments ";
|
|
}
|
|
|
|
if (&get("TRAINING:ghkm-phrase-orientation")) {
|
|
$cmd .= "-ghkm-phrase-orientation ";
|
|
my $phrase_orientation_priors_file = &versionize(&long_file_name("phrase-orientation-priors","model",""));
|
|
$cmd .= "-phrase-orientation-priors-file $phrase_orientation_priors_file ";
|
|
}
|
|
|
|
if (&get("TRAINING:ghkm-source-labels")) {
|
|
$cmd .= "-ghkm-source-labels ";
|
|
my $source_labels_file = &versionize(&long_file_name("source-labels","model",""));
|
|
$cmd .= "-ghkm-source-labels-file $source_labels_file ";
|
|
}
|
|
|
|
if (&get("TRAINING:ghkm-parts-of-speech")) {
|
|
$cmd .= "-ghkm-parts-of-speech ";
|
|
my $parts_of_speech_labels_file = &versionize(&long_file_name("parts-of-speech","model",""));
|
|
$cmd .= "-ghkm-parts-of-speech-file $parts_of_speech_labels_file ";
|
|
}
|
|
}
|
|
|
|
&create_step($step_id,$cmd);
|
|
}
|
|
|
|
sub define_domain_feature_score_option {
|
|
my ($domains) = @_;
|
|
my $spec = &backoff_and_get("TRAINING:domain-features");
|
|
my ($method,$restricted_to_table) = ("","");
|
|
$method = "Indicator" if $spec =~ /indicator/;
|
|
$method = "Ratio" if $spec =~ /ratio/;
|
|
$method = "Subset" if $spec =~ /subset/;
|
|
$restricted_to_table = $1 if $spec =~ /( table \S+)/;
|
|
die("ERROR: faulty TRAINING:domain-features spec (no method): $spec\n") unless defined($method);
|
|
if ($spec =~ /sparse/) {
|
|
return "-score-options '--SparseDomain$method $domains$restricted_to_table' ";
|
|
}
|
|
else {
|
|
return "-score-options '--Domain$method $domains' ";
|
|
}
|
|
}
|
|
|
|
sub define_training_build_reordering {
|
|
my ($step_id) = @_;
|
|
|
|
my ($reordering_table, $extract) = &get_output_and_input($step_id);
|
|
my $cmd = &get_training_setting(7);
|
|
$cmd .= "-extract-file $extract ";
|
|
$cmd .= &get_table_name_settings("reordering-factors","reordering-table",$reordering_table);
|
|
|
|
&create_step($step_id,$cmd);
|
|
}
|
|
|
|
sub define_training_build_generation {
|
|
my ($step_id) = @_;
|
|
|
|
my ($generation_table, $corpus) = &get_output_and_input($step_id);
|
|
my $cmd = &get_training_setting(8);
|
|
$cmd .= "-corpus $corpus ";
|
|
$cmd .= &get_table_name_settings("generation-factors","generation-table",$generation_table);
|
|
|
|
&create_step($step_id,$cmd);
|
|
}
|
|
|
|
sub define_training_build_custom_generation {
|
|
my ($step_id) = @_;
|
|
|
|
my ($generation_table, $generation_corpus) = &get_output_and_input($step_id);
|
|
my $cmd = &get_training_setting(8);
|
|
$cmd .= "-generation-corpus $generation_corpus ";
|
|
$cmd .= &get_table_name_settings("generation-factors","generation-table",$generation_table);
|
|
|
|
&create_step($step_id,$cmd);
|
|
}
|
|
|
|
sub define_training_sigtest_filter {
|
|
my ($step_id) = @_;
|
|
my ($filtered_table, $raw_table,$suffix_array) = &get_output_and_input($step_id);
|
|
|
|
my $hierarchical_flag = &get("TRAINING:hierarchical-rule-set") ? "-h" : "";
|
|
my $sigtest_filter = &get("TRAINING:sigtest-filter");
|
|
my $input_extension = &check_backoff_and_get("TRAINING:input-extension");
|
|
my $output_extension = &check_backoff_and_get("TRAINING:output-extension");
|
|
my $moses_src_dir = &check_and_get("GENERAL:moses-src-dir");
|
|
|
|
if ($DO_STEP[$step_id] =~ /reordering/) {
|
|
$raw_table = &get_table_name_settings("reordering-factors","reordering-table", $raw_table);
|
|
$filtered_table = &get_table_name_settings("reordering-factors","reordering-table", $filtered_table);
|
|
chop($raw_table);
|
|
chop($filtered_table);
|
|
$raw_table .= ".wbe-".&get("TRAINING:lexicalized-reordering"); # a bit of a hack
|
|
$filtered_table .= ".wbe-".&get("TRAINING:lexicalized-reordering");
|
|
}
|
|
else {
|
|
$raw_table = &get_table_name_settings("translation-factors","phrase-translation-table", $raw_table);
|
|
$filtered_table = &get_table_name_settings("translation-factors","phrase-translation-table", $filtered_table);
|
|
chop($raw_table);
|
|
chop($filtered_table);
|
|
}
|
|
$raw_table =~ s/\s*\-\S+\s*//; # remove switch
|
|
$filtered_table =~ s/\s*\-\S+\s*//;
|
|
|
|
my $cmd = "zcat $raw_table.gz | $moses_src_dir/contrib/sigtest-filter/filter-pt -e $suffix_array.$output_extension -f $suffix_array.$input_extension $sigtest_filter $hierarchical_flag | gzip - > $filtered_table.gz\n";
|
|
&create_step($step_id,$cmd);
|
|
}
|
|
|
|
sub get_config_tables {
|
|
my ($config,$reordering_table,$phrase_translation_table,$generation_table,$domains) = @_;
|
|
|
|
my $moses_src_dir = &check_and_get("GENERAL:moses-src-dir");
|
|
my $cmd = &get_training_setting(9);
|
|
|
|
# get model, and whether suffix array is used. Determines the pt implementation.
|
|
my $hierarchical = &get("TRAINING:hierarchical-rule-set");
|
|
$cmd .= "-hierarchical " if $hierarchical;
|
|
|
|
my $sa_exec_dir = &get("TRAINING:suffix-array");
|
|
my ($ptImpl, $numFF) = (0);
|
|
if ($hierarchical) {
|
|
if ($sa_exec_dir) {
|
|
$ptImpl = 10; # suffix array
|
|
$numFF = 7;
|
|
}
|
|
else {
|
|
$ptImpl = 6; # in-mem SCFG
|
|
}
|
|
}
|
|
|
|
# memory mapped suffix array phrase table
|
|
my $mmsapt = &get("TRAINING:mmsapt");
|
|
if (defined($mmsapt)) {
|
|
$ptImpl = 11; # mmsapt
|
|
$numFF = $1;
|
|
$cmd .= "-mmsapt ";
|
|
}
|
|
|
|
# additional settings for factored models
|
|
$cmd .= &get_table_name_settings("translation-factors","phrase-translation-table", $phrase_translation_table);
|
|
$cmd = trim($cmd);
|
|
$cmd .= ":$ptImpl" if $ptImpl>0;
|
|
$cmd .= ":$numFF" if defined($numFF);
|
|
$cmd .= " ";
|
|
|
|
$cmd .= &get_table_name_settings("reordering-factors","reordering-table",$reordering_table) if $reordering_table;
|
|
$cmd .= &get_table_name_settings("generation-factors","generation-table",$generation_table) if $generation_table;
|
|
$cmd .= "-config $config ";
|
|
|
|
my $decoding_graph_backoff = &get("TRAINING:decoding-graph-backoff");
|
|
if ($decoding_graph_backoff) {
|
|
$cmd .= "-decoding-graph-backoff \"$decoding_graph_backoff\" ";
|
|
}
|
|
|
|
# additional settings for hierarchical models
|
|
my $extract_version = $VERSION;
|
|
if (&get("TRAINING:hierarchical-rule-set")) {
|
|
$extract_version = $RE_USE[$STEP_LOOKUP{"TRAINING:extract-phrases"}]
|
|
if defined($STEP_LOOKUP{"TRAINING:extract-phrases"});
|
|
my $no_glue_grammar = &get("TRAINING:no-glue-grammar");
|
|
if (!defined($no_glue_grammar) || $no_glue_grammar eq "false") {
|
|
my $glue_grammar_file = &get("TRAINING:glue-grammar");
|
|
$glue_grammar_file = &versionize(&long_file_name("glue-grammar","model",""),$extract_version)
|
|
unless $glue_grammar_file;
|
|
$cmd .= "-glue-grammar-file $glue_grammar_file ";
|
|
}
|
|
if (&get("TRAINING:dont-tune-glue-grammar")) {
|
|
$cmd .= "-dont-tune-glue-grammar ";
|
|
}
|
|
if (&get("TRAINING:use-syntax-input-weight-feature")) {
|
|
$cmd .= "-use-syntax-input-weight-feature ";
|
|
}
|
|
}
|
|
|
|
# additional settings for syntax models
|
|
if (&get("GENERAL:output-parser") && (&get("TRAINING:use-unknown-word-labels") || &get("TRAINING:use-unknown-word-soft-matches"))) {
|
|
my $unknown_word_label = &versionize(&long_file_name("unknown-word-label","model",""),$extract_version);
|
|
$cmd .= "-unknown-word-label $unknown_word_label ";
|
|
}
|
|
if (&get("GENERAL:output-parser") && &get("TRAINING:use-unknown-word-soft-matches")) {
|
|
my $unknown_word_soft_matches = &versionize(&long_file_name("unknown-word-soft-matches","model",""),$extract_version);
|
|
$cmd .= "-unknown-word-soft-matches $unknown_word_soft_matches ";
|
|
}
|
|
# configuration due to domain features
|
|
$cmd .= &define_domain_feature_score_option($domains) if &get("TRAINING:domain-features");
|
|
# additional specified items from config
|
|
my $additional_ini = &get("TRAINING:additional-ini");
|
|
$cmd .= "-additional-ini '$additional_ini' " if defined($additional_ini);
|
|
|
|
return $cmd;
|
|
}
|
|
|
|
sub define_training_create_config {
|
|
my ($step_id) = @_;
|
|
|
|
my ($config,$reordering_table,$phrase_translation_table,$transliteration_pt,$generation_table,$sparse_lexical_features,$domains,$osm, @LM)
|
|
= &get_output_and_input($step_id);
|
|
|
|
my $cmd = &get_config_tables($config,$reordering_table,$phrase_translation_table,$generation_table,$domains);
|
|
|
|
if($transliteration_pt){
|
|
$cmd .= "-transliteration-phrase-table $transliteration_pt ";
|
|
}
|
|
|
|
if ($osm) {
|
|
my $osm_settings = &get("TRAINING:operation-sequence-model-settings");
|
|
if ($osm_settings =~ /-factor *(\S+)/){
|
|
$cmd .= "-osm-model $osm/ -osm-setting $1 ";
|
|
}
|
|
else {
|
|
$cmd .= "-osm-model $osm/operationLM.bin ";
|
|
}
|
|
}
|
|
|
|
if (&get("TRAINING:ghkm-source-labels")) {
|
|
$cmd .= "-ghkm-source-labels ";
|
|
my $source_labels_file = &versionize(&long_file_name("source-labels","model",""));
|
|
$cmd .= "-ghkm-source-labels-file $source_labels_file ";
|
|
}
|
|
|
|
if (&get("TRAINING:ghkm-parts-of-speech")) {
|
|
$cmd .= "-ghkm-parts-of-speech ";
|
|
my $parts_of_speech_labels_file = &versionize(&long_file_name("parts-of-speech","model",""));
|
|
$cmd .= "-ghkm-parts-of-speech-file $parts_of_speech_labels_file ";
|
|
}
|
|
|
|
# sparse lexical features provide additional content for config file
|
|
my @additional_ini_files;
|
|
push (@additional_ini_files, "$sparse_lexical_features.ini") if $sparse_lexical_features;
|
|
|
|
my @LM_SETS = &get_sets("LM");
|
|
my %INTERPOLATED_AWAY;
|
|
my %OUTPUT_FACTORS;
|
|
%OUTPUT_FACTORS = &get_factor_id("output") if &backoff_and_get("TRAINING:output-factors");
|
|
|
|
if (&get("INTERPOLATED-LM:script")) {
|
|
my $type = 0;
|
|
# binarizing the lm?
|
|
$type = 1 if (&get("INTERPOLATED-LM:binlm") ||
|
|
&backoff_and_get("INTERPOLATED-LM:lm-binarizer"));
|
|
# randomizing the lm?
|
|
$type = 5 if (&get("INTERPOLATED-LM:rlm") ||
|
|
&backoff_and_get("INTERPOLATED-LM:lm-randomizer"));
|
|
|
|
# manually set type
|
|
$type = &get("INTERPOLATED-LM:type") if &get("INTERPOLATED-LM:type");
|
|
|
|
# go through each interpolated language model
|
|
my ($icount,$ILM_SETS) = &get_interpolated_lm_sets();
|
|
my $FACTOR = &backoff_and_get_array("TRAINING:output-factors");
|
|
foreach my $factor (keys %{$ILM_SETS}) {
|
|
foreach my $order (keys %{$$ILM_SETS{$factor}}) {
|
|
next unless scalar(@{$$ILM_SETS{$factor}{$order}}) > 1;
|
|
my $suffix = "";
|
|
$suffix = ".$$FACTOR[$factor]" if $icount > 1 && defined($FACTOR);
|
|
$suffix .= ".order$order" if $icount > 1;
|
|
$cmd .= "-lm $factor:$order:$LM[0]$suffix:$type ";
|
|
foreach my $id_set (@{$$ILM_SETS{$factor}{$order}}) {
|
|
my ($id,$set) = split(/ /,$id_set,2);
|
|
$INTERPOLATED_AWAY{$set} = 1;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
shift @LM; # remove interpolated lm
|
|
|
|
my $feature_lines = "";
|
|
my $weight_lines = "";
|
|
|
|
|
|
die("ERROR: number of defined LM sets (".(scalar @LM_SETS).":".join(",",@LM_SETS).") and LM files (".(scalar @LM).":".join(",",@LM).") does not match")
|
|
unless scalar @LM == scalar @LM_SETS;
|
|
foreach my $lm (@LM) {
|
|
my $set = shift @LM_SETS;
|
|
next if defined($INTERPOLATED_AWAY{$set});
|
|
|
|
if (&get("LM:$set:config-feature-line") && &get("LM:$set:config-weight-line")) {
|
|
$feature_lines .= &get("LM:$set:config-feature-line") . ";";
|
|
$weight_lines .= &get("LM:$set:config-weight-line") . ";";
|
|
} elsif (&get("LM:$set:nplm")) {
|
|
push(@additional_ini_files, "$lm/nplm.ini");
|
|
} elsif (&get("LM:$set:bilingual-lm")) {
|
|
push(@additional_ini_files, "$lm/blm.ini");
|
|
} else {
|
|
my $order = &check_backoff_and_get("LM:$set:order");
|
|
|
|
my $lm_file = "$lm";
|
|
my $type = 0; # default: SRILM
|
|
|
|
# binarized language model?
|
|
$type = 1 if (&get("LM:$set:binlm") ||
|
|
&backoff_and_get("LM:$set:lm-binarizer"));
|
|
|
|
# using a randomized lm?
|
|
$type = 5 if (&get("LM:$set:rlm") ||
|
|
&backoff_and_get("LM:$set:rlm-training") ||
|
|
&backoff_and_get("LM:$set:lm-randomizer"));
|
|
|
|
# manually set type
|
|
$type = &backoff_and_get("LM:$set:type") if (&backoff_and_get("LM:$set:type"));
|
|
|
|
# which factor is the model trained on?
|
|
my $factor = 0;
|
|
if (&backoff_and_get("TRAINING:output-factors") &&
|
|
&backoff_and_get("LM:$set:factors")) {
|
|
$factor = $OUTPUT_FACTORS{&backoff_and_get("LM:$set:factors")};
|
|
}
|
|
|
|
$cmd .= "-lm $factor:$order:$lm_file:$type ";
|
|
}
|
|
}
|
|
|
|
if ($feature_lines) {
|
|
$cmd .= "-config-add-feature-lines \"$feature_lines\" ";
|
|
}
|
|
if ($weight_lines) {
|
|
$cmd .= "-config-add-weight-lines \"$weight_lines\" ";
|
|
}
|
|
|
|
if (@additional_ini_files) {
|
|
$cmd .= "-additional-ini-file " . join(":", @additional_ini_files);
|
|
}
|
|
|
|
&create_step($step_id,$cmd);
|
|
}
|
|
|
|
sub define_interpolated_lm_interpolate {
|
|
my ($step_id) = @_;
|
|
|
|
my ($interpolated_lm,
|
|
$interpolation_script, $tuning, @LM) = &get_output_and_input($step_id);
|
|
my $srilm_dir = &check_backoff_and_get("INTERPOLATED-LM:srilm-dir");
|
|
my $group = &get("INTERPOLATED-LM:group");
|
|
my $weights = &get("INTERPOLATED-LM:weights");
|
|
my $scripts = &check_backoff_and_get("TUNING:moses-script-dir");
|
|
|
|
my $cmd = "";
|
|
|
|
my %WEIGHT;
|
|
if (defined($weights)) {
|
|
foreach (split(/ *, */,$weights)) {
|
|
/^ *(\S+) *= *(\S+)/ || die("ERROR: wrong interpolation weight specification $_ ($weights)");
|
|
$WEIGHT{$1} = $2;
|
|
}
|
|
}
|
|
|
|
# go through language models by factor and order
|
|
my ($icount,$ILM_SETS) = &get_interpolated_lm_sets();
|
|
foreach my $factor (keys %{$ILM_SETS}) {
|
|
foreach my $order (keys %{$$ILM_SETS{$factor}}) {
|
|
next unless scalar(@{$$ILM_SETS{$factor}{$order}}) > 1;
|
|
|
|
# get list of language model files
|
|
my $lm_list = "";
|
|
my $weight_list = "";
|
|
foreach my $id_set (@{$$ILM_SETS{$factor}{$order}}) {
|
|
my ($id,$set) = split(/ /,$id_set,2);
|
|
$lm_list .= $LM[$id]."," if $LM[$id];
|
|
if (defined($weights)) {
|
|
die("ERROR: no interpolation weight set for $factor:$order:$set (factor:order:set)")
|
|
unless defined($WEIGHT{"$factor:$order:$set"});
|
|
$weight_list .= $WEIGHT{"$factor:$order:$set"}.",";
|
|
}
|
|
}
|
|
chop($lm_list);
|
|
chop($weight_list);
|
|
|
|
# if grouping, identify position in list
|
|
my $numbered_string = "";
|
|
if (defined($group)) {
|
|
my %POSITION;
|
|
foreach my $id_set (@{$$ILM_SETS{$factor}{$order}}) {
|
|
my ($id,$set) = split(/ /,$id_set,2);
|
|
$POSITION{$set} = scalar keys %POSITION;
|
|
}
|
|
my $group_string = $group;
|
|
$group_string =~ s/\s+/ /g;
|
|
$group_string =~ s/ *, */,/g;
|
|
$group_string =~ s/^ //;
|
|
$group_string =~ s/ $//;
|
|
$group_string .= " ";
|
|
while($group_string =~ /^([^ ,]+)([ ,]+)(.*)$/) {
|
|
# die("ERROR: unknown set $1 in INTERPOLATED-LM:group definition")
|
|
# if ! defined($POSITION{$1});
|
|
# detect that elsewhere!
|
|
if (defined($POSITION{$1})) {
|
|
$numbered_string .= $POSITION{$1}.$2;
|
|
}
|
|
$group_string = $3;
|
|
}
|
|
chop($numbered_string);
|
|
}
|
|
|
|
my $FACTOR = &backoff_and_get_array("TRAINING:output-factors");
|
|
my $name = $interpolated_lm;
|
|
if ($icount > 1) {
|
|
$name .= ".$$FACTOR[$factor]" if defined($FACTOR);
|
|
$name .= ".order$order";
|
|
}
|
|
my $factored_tuning = $tuning;
|
|
if (&backoff_and_get("TRAINING:output-factors")) {
|
|
$factored_tuning = "$tuning.factor$factor";
|
|
$cmd .= "$scripts/training/reduce-factors.perl --corpus $tuning --reduced $factored_tuning --factor $factor\n";
|
|
}
|
|
$cmd .= "$interpolation_script --tuning $factored_tuning --name $name --srilm $srilm_dir --lm $lm_list";
|
|
$cmd .= " --group \"$numbered_string\"" if defined($group);
|
|
$cmd .= " --weights \"$weight_list\"" if defined($weights);
|
|
$cmd .= "\n";
|
|
}
|
|
}
|
|
|
|
die("ERROR: Nothing to interpolate, remove interpolation step!") if $cmd eq "";
|
|
&create_step($step_id,$cmd);
|
|
}
|
|
|
|
sub define_interpolated_lm_process {
|
|
my ($step_id) = @_;
|
|
|
|
my ($processed_lm, $interpolated_lm) = &get_output_and_input($step_id);
|
|
my ($module,$set,$stepname) = &deconstruct_name($DO_STEP[$step_id]);
|
|
my $tool = &check_backoff_and_get("INTERPOLATED-LM:lm-${stepname}r");
|
|
my $FACTOR = &backoff_and_get_array("TRAINING:output-factors");
|
|
|
|
# go through language models by factor and order
|
|
my ($icount,$ILM_SETS) = &get_interpolated_lm_sets();
|
|
my $cmd = "";
|
|
foreach my $factor (keys %{$ILM_SETS}) {
|
|
foreach my $order (keys %{$$ILM_SETS{$factor}}) {
|
|
my ($name,$name_processed);
|
|
if (scalar(@{$$ILM_SETS{$factor}{$order}}) == 1) {
|
|
# not interpolated -> get name from LM version of these steps
|
|
my($id,$set) = split(/ /,$$ILM_SETS{$factor}{$order}[0]);
|
|
$name = &get_default_file("LM",$set,"train"); # well... works for now;
|
|
$name_processed = $STEP_OUTNAME{"LM:$stepname"};
|
|
$name_processed =~ s/^(.+\/)([^\/]+)$/$1$set.$2/;
|
|
$name_processed = &versionize(&long_file_name($name_processed,"lm",""));
|
|
}
|
|
else {
|
|
my $suffix = "";
|
|
$suffix = ".$$FACTOR[$factor]" if $icount > 1 && defined($FACTOR);
|
|
$suffix .= ".order$order" if $icount > 1;
|
|
$name = "$interpolated_lm$suffix";
|
|
$name_processed = "$processed_lm$suffix";
|
|
}
|
|
$cmd .= "$tool $name $name_processed\n";
|
|
}
|
|
}
|
|
|
|
&create_step($step_id,$cmd);
|
|
}
|
|
|
|
sub get_interpolated_lm_processed_names {
|
|
my ($processed_lm) = @_;
|
|
my @ILM_NAME;
|
|
my ($icount,$ILM_SETS) = &get_interpolated_lm_sets();
|
|
my $FACTOR = &backoff_and_get_array("TRAINING:output-factors");
|
|
foreach my $factor (keys %{$ILM_SETS}) {
|
|
foreach my $order (keys %{$$ILM_SETS{$factor}}) {
|
|
if (scalar(@{$$ILM_SETS{$factor}{$order}}) > 1) {
|
|
my $suffix = "";
|
|
$suffix = ".$$FACTOR[$factor]" if $icount > 1 && defined($FACTOR);
|
|
$suffix .= ".order$order" if $icount > 1;
|
|
push @ILM_NAME,"$processed_lm$suffix";
|
|
}
|
|
else {
|
|
push @ILM_NAME,"$processed_lm.".($FACTOR?"":".$$FACTOR[$factor]").".order$order";
|
|
}
|
|
}
|
|
}
|
|
return @ILM_NAME;
|
|
}
|
|
|
|
sub get_interpolated_lm_sets {
|
|
my %ILM_SETS;
|
|
|
|
my @LM_SETS = &get_sets("LM");
|
|
my %OUTPUT_FACTORS;
|
|
%OUTPUT_FACTORS = &get_factor_id("output") if &backoff_and_get("TRAINING:output-factors");
|
|
|
|
my $count=0;
|
|
my $icount=0;
|
|
foreach my $set (@LM_SETS) {
|
|
next if (&get("LM:$set:exclude-from-interpolation")) or (&get("LM:$set:bilingual-lm"))
|
|
or (&get("LM:$set:nplm"));
|
|
my $order = &check_backoff_and_get("LM:$set:order");
|
|
|
|
my $factor = 0;
|
|
if (&backoff_and_get("TRAINING:output-factors") &&
|
|
&backoff_and_get("LM:$set:factors")) {
|
|
$factor = $OUTPUT_FACTORS{&backoff_and_get("LM:$set:factors")};
|
|
}
|
|
|
|
push @{$ILM_SETS{$factor}{$order}}, ($count++)." ".$set;
|
|
$icount++ if scalar(@{$ILM_SETS{$factor}{$order}}) == 2;
|
|
}
|
|
return ($icount,\%ILM_SETS);
|
|
}
|
|
|
|
sub get_training_setting {
|
|
my ($step) = @_;
|
|
my $dir = &check_and_get("GENERAL:working-dir");
|
|
my $training_script = &check_and_get("TRAINING:script");
|
|
my $external_bin_dir = &check_backoff_and_get("TRAINING:external-bin-dir");
|
|
my $scripts = &check_backoff_and_get("TUNING:moses-script-dir");
|
|
my $reordering = &get("TRAINING:lexicalized-reordering");
|
|
my $input_extension = &check_backoff_and_get("TRAINING:input-extension");
|
|
my $output_extension = &check_backoff_and_get("TRAINING:output-extension");
|
|
my $alignment = &check_and_get("TRAINING:alignment-symmetrization-method");
|
|
my $parts = &get("TRAINING:run-giza-in-parts");
|
|
my $options = &get("TRAINING:training-options");
|
|
my $phrase_length = &get("TRAINING:max-phrase-length");
|
|
my $hierarchical = &get("TRAINING:hierarchical-rule-set");
|
|
my $source_syntax = &get("GENERAL:input-parser");
|
|
my $target_syntax = &get("GENERAL:output-parser");
|
|
my $score_settings = &get("TRAINING:score-settings");
|
|
my $parallel = &get("TRAINING:parallel");
|
|
my $pcfg = &get("TRAINING:use-pcfg-feature");
|
|
my $baseline_alignment = &get("TRAINING:baseline-alignment-model");
|
|
my $no_glue_grammar = &get("TRAINING:no-glue-grammar");
|
|
my $mmsapt = &get("TRAINING:mmsapt");
|
|
|
|
my $xml = $source_syntax || $target_syntax;
|
|
|
|
my $cmd = "$training_script ";
|
|
$cmd .= "$options " if defined($options);
|
|
$cmd .= "-dont-zip ";
|
|
$cmd .= "-first-step $step " if $step>1;
|
|
$cmd .= "-last-step $step " if $step<9;
|
|
$cmd .= "-external-bin-dir $external_bin_dir " if defined($external_bin_dir);
|
|
$cmd .= "-f $input_extension -e $output_extension ";
|
|
$cmd .= "-alignment $alignment ";
|
|
$cmd .= "-max-phrase-length $phrase_length " if $phrase_length;
|
|
$cmd .= "-parts $parts " if $parts;
|
|
$cmd .= "-reordering $reordering " if $reordering;
|
|
$cmd .= "-temp-dir /disk/scratch2 " if `hostname` =~ /townhill/;
|
|
$cmd .= "-hierarchical " if $hierarchical;
|
|
$cmd .= "-xml " if $xml;
|
|
$cmd .= "-target-syntax " if $target_syntax;
|
|
$cmd .= "-source-syntax " if $source_syntax;
|
|
$cmd .= "-glue-grammar " if $hierarchical && (!defined($no_glue_grammar) || $no_glue_grammar eq "false");
|
|
$cmd .= "-score-options '".$score_settings."' " if $score_settings;
|
|
$cmd .= "-parallel " if $parallel;
|
|
$cmd .= "-pcfg " if $pcfg;
|
|
$cmd .= "-baseline-alignment-model $baseline_alignment " if defined($baseline_alignment) && ($step == 1 || $step == 2);
|
|
$cmd .= "-mmsapt " if defined($mmsapt);
|
|
|
|
# factored training
|
|
if (&backoff_and_get("TRAINING:input-factors")) {
|
|
my %IN = &get_factor_id("input");
|
|
my %OUT = &get_factor_id("output");
|
|
$cmd .= "-input-factor-max ".((scalar keys %IN)-1)." ";
|
|
$cmd .= "-alignment-factors ".
|
|
&encode_factor_definition("alignment-factors",\%IN,\%OUT)." ";
|
|
$cmd .= "-translation-factors ".
|
|
&encode_factor_definition("translation-factors",\%IN,\%OUT)." ";
|
|
$cmd .= "-reordering-factors ".
|
|
&encode_factor_definition("reordering-factors",\%IN,\%OUT)." "
|
|
if &get("TRAINING:reordering-factors");
|
|
$cmd .= "-generation-factors ".
|
|
&encode_factor_definition("generation-factors",\%OUT,\%OUT)." "
|
|
if &get("TRAINING:generation-factors");
|
|
die("ERROR: define either both TRAINING:reordering-factors and TRAINING:reordering or neither")
|
|
if (( &get("TRAINING:reordering-factors") && ! $reordering) ||
|
|
(! &get("TRAINING:reordering-factors") && $reordering));
|
|
my $decoding_steps = &check_and_get("TRAINING:decoding-steps");
|
|
$decoding_steps =~ s/\s*//g;
|
|
$cmd .= "-decoding-steps $decoding_steps ";
|
|
my $generation_type = &get("TRAINING:generation-type");
|
|
$cmd .= "-generation-type $generation_type " if $generation_type;
|
|
}
|
|
|
|
return $cmd;
|
|
}
|
|
|
|
sub get_table_name_settings {
|
|
my ($factor,$table,$default) = @_;
|
|
my $dir = &check_and_get("GENERAL:working-dir");
|
|
|
|
my @NAME;
|
|
if (!&backoff_and_get("TRAINING:input-factors")) {
|
|
return "-$table $default ";
|
|
}
|
|
|
|
# define default names
|
|
my %IN = &get_factor_id("input");
|
|
my %OUT = &get_factor_id("output");
|
|
%IN = %OUT if $factor eq "generation-factors";
|
|
my $factors = &encode_factor_definition($factor,\%IN,\%OUT);
|
|
foreach my $f (split(/\+/,$factors)) {
|
|
push @NAME,"$default.$f";
|
|
# push @NAME,"$dir/model/$table.$VERSION.$f";
|
|
}
|
|
|
|
# get specified names, if any
|
|
if (&get("TRAINING:$table")) {
|
|
my @SPECIFIED_NAME = @{$CONFIG{"TRAINING:$table"}};
|
|
die("ERROR: specified more ${table}s than $factor")
|
|
if (scalar @SPECIFIED_NAME) > (scalar @NAME);
|
|
for(my $i=0;$i<scalar(@SPECIFIED_NAME);$i++) {
|
|
$NAME[$i] = $SPECIFIED_NAME[$i];
|
|
}
|
|
}
|
|
|
|
# create command
|
|
my $cmd;
|
|
foreach my $name (@NAME) {
|
|
$cmd .= "-$table $name ";
|
|
}
|
|
return $cmd;
|
|
}
|
|
|
|
sub get_factor_id {
|
|
my ($type) = @_;
|
|
my $FACTOR = &check_backoff_and_get_array("TRAINING:$type-factors");
|
|
my %ID = ();
|
|
foreach my $factor (@{$FACTOR}) {
|
|
$ID{$factor} = scalar keys %ID;
|
|
}
|
|
return %ID;
|
|
}
|
|
|
|
sub encode_factor_definition {
|
|
my ($parameter,$IN,$OUT) = @_;
|
|
my $definition = &check_and_get("TRAINING:$parameter");
|
|
my $encoded;
|
|
foreach my $mapping (split(/,\s*/,$definition)) {
|
|
my ($in,$out) = split(/\s*->\s*/,$mapping);
|
|
$encoded .=
|
|
&encode_factor_list($IN,$in)."-".
|
|
&encode_factor_list($OUT,$out)."+";
|
|
}
|
|
chop($encoded);
|
|
return $encoded;
|
|
}
|
|
|
|
sub encode_factor_list {
|
|
my ($ID,$list) = @_;
|
|
my $id;
|
|
foreach my $factor (split(/\s*\+\s*/,$list)) {
|
|
die("ERROR: unknown factor type '$factor'\n") unless defined($$ID{$factor});
|
|
$id .= $$ID{$factor}.",";
|
|
}
|
|
chop($id);
|
|
return $id;
|
|
}
|
|
|
|
sub define_tuningevaluation_filter {
|
|
my ($set,$step_id) = @_;
|
|
my $scripts = &check_and_get("GENERAL:moses-script-dir");
|
|
my $dir = &check_and_get("GENERAL:working-dir");
|
|
my $word_alignment = &backoff_and_get("TRAINING:include-word-alignment-in-rules");
|
|
my $tuning_flag = !defined($set);
|
|
my $hierarchical = &get("TRAINING:hierarchical-rule-set");
|
|
|
|
my ($filter_dir,$input,$phrase_translation_table,$reordering_table,$domains,$transliteration_table) = &get_output_and_input($step_id);
|
|
|
|
my $binarizer;
|
|
$binarizer = &backoff_and_get("EVALUATION:$set:ttable-binarizer") unless $tuning_flag;
|
|
$binarizer = &backoff_and_get("TUNING:ttable-binarizer") if $tuning_flag;
|
|
my $report_precision_by_coverage = !$tuning_flag && &backoff_and_get("EVALUATION:$set:report-precision-by-coverage");
|
|
|
|
# occasionally, lattices and conf nets need to be able
|
|
# to filter phrase tables, we can provide sentences/ngrams
|
|
# in a separate file
|
|
my $input_filter;
|
|
$input_filter = &get("EVALUATION:$set:input-filter") unless $tuning_flag;
|
|
$input_filter = &get("TUNING:input-filter") if $tuning_flag;
|
|
#print "filter: $input_filter \n";
|
|
$input_filter = $input unless $input_filter;
|
|
|
|
my $settings = &backoff_and_get("EVALUATION:$set:filter-settings") unless $tuning_flag;
|
|
$settings = &backoff_and_get("TUNING:filter-settings") if $tuning_flag;
|
|
$settings = "" unless $settings;
|
|
|
|
$binarizer .= " -no-alignment-info" if defined ($binarizer) && !$hierarchical && defined $word_alignment && $word_alignment eq "no";
|
|
|
|
$settings .= " -Binarizer \"$binarizer\"" if $binarizer;
|
|
$settings .= " --Hierarchical" if $hierarchical;
|
|
|
|
# get model, and whether suffix array is used. Determines the pt implementation.
|
|
my $sa_exec_dir = &get("TRAINING:suffix-array");
|
|
my $sa_extractors = &get("GENERAL:sa_extractors");
|
|
$sa_extractors = 1 unless $sa_extractors;
|
|
|
|
my ($ptImpl, $numFF);
|
|
if ($hierarchical) {
|
|
if ($sa_exec_dir) {
|
|
$ptImpl = 10; # suffix array
|
|
$numFF = 7;
|
|
}
|
|
else {
|
|
$ptImpl = 6; # in-mem SCFG
|
|
}
|
|
}
|
|
else {
|
|
$ptImpl = 0; # phrase-based
|
|
}
|
|
|
|
# config file specified?
|
|
my ($config,$cmd,$delete_config);
|
|
if (&get("TUNING:config-with-reused-weights")) {
|
|
$config = &get("TUNING:config-with-reused-weights");
|
|
}
|
|
elsif (&get("TRAINING:config")) {
|
|
$config = &get("TRAINING:config");
|
|
}
|
|
# create pseudo-config file
|
|
else {
|
|
$config = $tuning_flag ? "$dir/tuning/moses.table.ini.$VERSION" : "$dir/evaluation/$set.moses.table.ini.$VERSION";
|
|
$cmd = "touch $config\n";
|
|
$delete_config = 1;
|
|
|
|
$cmd .= &get_config_tables($config,$reordering_table,$phrase_translation_table,undef,$domains);
|
|
|
|
if (&get("TRAINING:in-decoding-transliteration")) {
|
|
|
|
$cmd .= "-transliteration-phrase-table $dir/model/transliteration-phrase-table.$VERSION ";
|
|
}
|
|
|
|
|
|
$cmd .= "-lm 0:3:$config:8\n"; # dummy kenlm 3-gram model on factor 0
|
|
|
|
}
|
|
|
|
# filter command
|
|
if ($sa_exec_dir) {
|
|
# suffix array
|
|
$cmd .= "$scripts/training/wrappers/adam-suffix-array/suffix-array-extract.sh $sa_exec_dir $phrase_translation_table $input_filter $filter_dir $sa_extractors \n";
|
|
|
|
my $escaped_filter_dir = $filter_dir;
|
|
$escaped_filter_dir =~ s/\//\\\\\//g;
|
|
$cmd .= "cat $config | sed s/10\\ 0\\ 0\\ 7.*/10\\ 0\\ 0\\ 7\\ $escaped_filter_dir/g > $filter_dir/moses.ini \n";
|
|
# kind of a hack -- the correct thing would be to make the generation of the config file ($filter_dir/moses.ini)
|
|
# set the PhraseDictionaryALSuffixArray's path to the filtered directory rather than to the suffix array itself
|
|
$cmd .= "sed -i 's%path=$phrase_translation_table%path=$filter_dir%' $filter_dir/moses.ini\n";
|
|
}
|
|
else {
|
|
# normal phrase table
|
|
$cmd .= "$scripts/training/filter-model-given-input.pl";
|
|
$cmd .= " $filter_dir $config $input_filter $settings\n";
|
|
}
|
|
|
|
# clean-up
|
|
$cmd .= "rm $config" if $delete_config;
|
|
|
|
&create_step($step_id,$cmd);
|
|
}
|
|
|
|
sub define_evaluation_decode {
|
|
my ($set,$step_id) = @_;
|
|
my $scripts = &check_and_get("GENERAL:moses-script-dir");
|
|
my $dir = &check_and_get("GENERAL:working-dir");
|
|
|
|
my ($system_output,
|
|
$config,$input,$filtered_config) = &get_output_and_input($step_id);
|
|
$config = $filtered_config if $filtered_config;
|
|
|
|
my $jobs = &backoff_and_get("EVALUATION:$set:jobs");
|
|
my $decoder = &check_backoff_and_get("EVALUATION:$set:decoder");
|
|
my $settings = &backoff_and_get("EVALUATION:$set:decoder-settings");
|
|
$settings = "" unless $settings;
|
|
my $nbest = &backoff_and_get("EVALUATION:$set:nbest");
|
|
my $moses_parallel = &backoff_and_get("EVALUATION:$set:moses-parallel");
|
|
my $report_segmentation = &backoff_and_get("EVALUATION:$set:report-segmentation");
|
|
my $analyze_search_graph = &backoff_and_get("EVALUATION:$set:analyze-search-graph");
|
|
my $report_precision_by_coverage = &backoff_and_get("EVALUATION:$set:report-precision-by-coverage");
|
|
my $use_wade = &backoff_and_get("EVALUATION:$set:wade");
|
|
my $hierarchical = &get("TRAINING:hierarchical-rule-set");
|
|
my $word_alignment = &backoff_and_get("TRAINING:include-word-alignment-in-rules");
|
|
my $post_decoding_transliteration = &get("TRAINING:post-decoding-transliteration");
|
|
|
|
# specify additional output for analysis
|
|
if (defined($report_precision_by_coverage) && $report_precision_by_coverage eq "yes") {
|
|
$settings .= " -alignment-output-file $system_output.wa";
|
|
$report_segmentation = "yes";
|
|
}
|
|
if (defined($analyze_search_graph) && $analyze_search_graph eq "yes") {
|
|
$settings .= " -unpruned-search-graph -include-lhs-in-search-graph -osg $system_output.graph";
|
|
}
|
|
if (defined($report_segmentation) && $report_segmentation eq "yes") {
|
|
if ($hierarchical) {
|
|
$settings .= " -T $system_output.trace";
|
|
}
|
|
else {
|
|
$settings .= " -t";
|
|
}
|
|
}
|
|
if ($use_wade) {
|
|
$settings .= " -T $system_output.details";
|
|
}
|
|
$settings .= " -text-type \"test\"";
|
|
|
|
my $addTags = &backoff_and_get("EVALUATION:$set:add-tags");
|
|
if ($addTags) {
|
|
my $input_with_tags = $input.".".$VERSION.".tags";
|
|
`$addTags < $input > $input_with_tags`;
|
|
$input = $input_with_tags;
|
|
}
|
|
|
|
# cache model on local disk
|
|
my $cmd;
|
|
my $cache_model = &backoff_and_get("GENERAL:cache-model");
|
|
if (defined($cache_model) && !($jobs && $jobs>1 && $CLUSTER)) {
|
|
my $scripts = &check_and_get("GENERAL:moses-script-dir");
|
|
$cmd = "MOSES_INI=`$scripts/ems/support/cache-model.perl $config $cache_model`\n";
|
|
$config = "\$MOSES_INI";
|
|
}
|
|
|
|
# create command
|
|
my $nbest_size;
|
|
$nbest_size = $nbest if $nbest;
|
|
$nbest_size =~ s/[^\d]//g if $nbest;
|
|
if ($jobs && $jobs>1 && $CLUSTER) {
|
|
$cmd .= "mkdir -p $dir/evaluation/tmp.$set.$VERSION\n";
|
|
$cmd .= "cd $dir/evaluation/tmp.$set.$VERSION\n";
|
|
if (defined $moses_parallel) {
|
|
$cmd .= $moses_parallel;
|
|
} else {
|
|
$cmd .= "$scripts/generic/moses-parallel.pl";
|
|
}
|
|
my $qsub_args = &get_qsub_args($DO_STEP[$step_id]);
|
|
$cmd .= " -queue-parameters \"$qsub_args\"" if ($CLUSTER && $qsub_args);
|
|
$cmd .= " -decoder $decoder";
|
|
$cmd .= " -config $config";
|
|
$cmd .= " -cache-model $cache_model" if defined($cache_model);
|
|
$cmd .= " -input-file $input";
|
|
$cmd .= " --jobs $jobs";
|
|
$cmd .= " -decoder-parameters \"$settings\" > $system_output";
|
|
$cmd .= " -n-best-file $system_output.best$nbest_size -n-best-size $nbest" if $nbest;
|
|
}
|
|
else {
|
|
$cmd = "$decoder $settings -v 0 -f $config < $input > $system_output";
|
|
$cmd .= " -n-best-list $system_output.best$nbest_size $nbest" if $nbest;
|
|
}
|
|
|
|
# If Transliteration Module is to be used as post-decoding step ...
|
|
$cmd .= " -output-unknowns $system_output.oov"
|
|
if defined($post_decoding_transliteration) && $post_decoding_transliteration eq "yes";
|
|
|
|
&create_step($step_id,$cmd);
|
|
}
|
|
|
|
sub define_evaluation_analysis {
|
|
my ($set,$step_id) = @_;
|
|
|
|
my ($analysis,
|
|
$output,$reference,$input) = &get_output_and_input($step_id);
|
|
my $script = &backoff_and_get("EVALUATION:$set:analysis");
|
|
my $report_segmentation = &backoff_and_get("EVALUATION:$set:report-segmentation");
|
|
my $analyze_search_graph = &backoff_and_get("EVALUATION:$set:analyze-search-graph");
|
|
|
|
my $cmd = "$script -system $output -reference $reference -input $input -dir $analysis";
|
|
if (defined($report_segmentation) && $report_segmentation eq "yes") {
|
|
my $segmentation_file = &get_default_file("EVALUATION",$set,"decode");
|
|
$cmd .= " -segmentation $segmentation_file";
|
|
}
|
|
if (defined($analyze_search_graph) && $analyze_search_graph eq "yes") {
|
|
my $search_graph_file = &get_default_file("EVALUATION",$set,"decode");
|
|
$cmd .= " -search-graph $search_graph_file.graph";
|
|
}
|
|
if (&get("TRAINING:hierarchical-rule-set")) {
|
|
$cmd .= " -hierarchical";
|
|
}
|
|
&create_step($step_id,$cmd);
|
|
}
|
|
|
|
sub define_evaluation_analysis_precision {
|
|
my ($set,$step_id) = @_;
|
|
|
|
my ($analysis,
|
|
$output,$reference,$input,$corpus,$ttable,$coverage) = &get_output_and_input($step_id);
|
|
my $script = &backoff_and_get("EVALUATION:$set:analysis");
|
|
my $input_extension = &check_backoff_and_get("TRAINING:input-extension");
|
|
my $coverage_base = &backoff_and_get("EVALUATION:$set:precision-by-coverage-base");
|
|
my $cmd = "$script -system $output -reference $reference -input $input -dir $analysis -precision-by-coverage";
|
|
|
|
my $segmentation_file = &get_default_file("EVALUATION",$set,"decode");
|
|
$cmd .= " -segmentation $segmentation_file";
|
|
$cmd .= " -system-alignment $segmentation_file.wa";
|
|
$coverage = $coverage_base if defined($coverage_base);
|
|
$cmd .= " -coverage $coverage";
|
|
|
|
# get table with surface factors
|
|
if (&backoff_and_get("TRAINING:input-factors")) {
|
|
my %IN = &get_factor_id("input");
|
|
my %OUT = &get_factor_id("output");
|
|
my $factors = &encode_factor_definition("translation-factors",\%IN,\%OUT);
|
|
my @FACTOR = split(/\+/,$factors);
|
|
my @SPECIFIED_NAME;
|
|
if (&backoff_and_get("TRAINING:sigtest-filter-phrase-translation-table")) {
|
|
@SPECIFIED_NAME = @{$CONFIG{"TRAINING:sigtest-filter-phrase-translation-table"}};
|
|
}
|
|
elsif (&backoff_and_get("TRAINING:phrase-translation-table")) {
|
|
@SPECIFIED_NAME = @{$CONFIG{"TRAINING:phrase-translation-table"}};
|
|
}
|
|
for(my $i=0;$i<scalar(split(/\+/,$factors));$i++) {
|
|
if ($FACTOR[$i] =~ /^0-/) {
|
|
if (scalar(@SPECIFIED_NAME) > $i) {
|
|
$ttable = $SPECIFIED_NAME[$i];
|
|
}
|
|
else {
|
|
$ttable .= ".".$FACTOR[$i];
|
|
}
|
|
last;
|
|
}
|
|
}
|
|
my $subreport = &backoff_and_get("EVALUATION:precision-by-coverage-factor");
|
|
if (defined($subreport)) {
|
|
die("unknown factor $subreport specified in EVALUATION:precision-by-coverage-factor") unless defined($IN{$subreport});
|
|
$cmd .= " -precision-by-coverage-factor ".$IN{$subreport};
|
|
}
|
|
}
|
|
$cmd .= " -ttable $ttable -input-corpus $corpus.$input_extension";
|
|
|
|
&create_step($step_id,$cmd);
|
|
}
|
|
|
|
sub define_evaluation_analysis_coverage {
|
|
my ($set,$step_id) = @_;
|
|
|
|
my ($analysis,
|
|
$input,$corpus,$ttable) = &get_output_and_input($step_id);
|
|
my $script = &backoff_and_get("EVALUATION:$set:analysis");
|
|
my $input_extension = &check_backoff_and_get("TRAINING:input-extension");
|
|
my $score_settings = &get("TRAINING:score-settings");
|
|
|
|
my $ttable_config;
|
|
|
|
# translation tables for un-factored
|
|
if (!&backoff_and_get("TRAINING:input-factors")) {
|
|
$ttable_config = "-ttable $ttable";
|
|
}
|
|
# translation tables for factored
|
|
else {
|
|
my %IN = &get_factor_id("input");
|
|
$ttable_config = "-input-factors ".(scalar(keys %IN));
|
|
my %OUT = &get_factor_id("output");
|
|
$ttable_config .= " -input-factor-names '".join(",",keys %IN)."'";
|
|
$ttable_config .= " -output-factor-names '".join(",",keys %OUT)."'";
|
|
my $factors = &encode_factor_definition("translation-factors",\%IN,\%OUT);
|
|
my @FACTOR = split(/\+/,$factors);
|
|
my @SPECIFIED_NAME;
|
|
if (&backoff_and_get("TRAINING:sigtest-filter-phrase-translation-table")) {
|
|
@SPECIFIED_NAME = @{$CONFIG{"TRAINING:sigtest-filter-phrase-translation-table"}};
|
|
}
|
|
elsif (&backoff_and_get("TRAINING:phrase-translation-table")) {
|
|
@SPECIFIED_NAME = @{$CONFIG{"TRAINING:phrase-translation-table"}};
|
|
}
|
|
my $surface_ttable;
|
|
for(my $i=0;$i<scalar(@FACTOR);$i++) {
|
|
$FACTOR[$i] =~ /^([\d\,]+)/;
|
|
my $input_factors = $1;
|
|
|
|
my $ttable_name = $ttable.".".$FACTOR[$i];
|
|
if (scalar(@SPECIFIED_NAME) > $i) {
|
|
$ttable_name = $SPECIFIED_NAME[$i];
|
|
}
|
|
|
|
$ttable_config .= " -factored-ttable $input_factors:".$ttable_name;
|
|
if ($input_factors eq "0" && !defined($surface_ttable)) {
|
|
$surface_ttable = $ttable_name;
|
|
$ttable_config .= " -ttable $surface_ttable";
|
|
}
|
|
}
|
|
}
|
|
|
|
my $cmd = "$script -input $input -input-corpus $corpus.$input_extension $ttable_config -dir $analysis";
|
|
$cmd .= " -score-options '$score_settings'" if $score_settings;
|
|
&create_step($step_id,$cmd);
|
|
}
|
|
|
|
sub define_reporting_report {
|
|
my ($step_id) = @_;
|
|
|
|
my $score_file = &get_default_file("REPORTING","","report");
|
|
|
|
my $scripts = &check_and_get("GENERAL:moses-script-dir");
|
|
my $cmd = "$scripts/ems/support/report-experiment-scores.perl";
|
|
|
|
# get scores that were produced
|
|
foreach my $parent (@{$DEPENDENCY[$step_id]}) {
|
|
my ($parent_module,$parent_set,$parent_step)
|
|
= &deconstruct_name($DO_STEP[$parent]);
|
|
|
|
my $file = &get_default_file($parent_module,$parent_set,$parent_step);
|
|
$cmd .= " set=$parent_set,type=$parent_step,file=$file";
|
|
}
|
|
|
|
# maybe send email
|
|
my $email = &get("REPORTING:email");
|
|
if ($email) {
|
|
$cmd .= " email='$email'";
|
|
}
|
|
|
|
$cmd .= " >$score_file";
|
|
|
|
&create_step($step_id,$cmd);
|
|
}
|
|
|
|
### subs for step definition
|
|
|
|
sub get_output_and_input {
|
|
my ($step_id) = @_;
|
|
|
|
my $step = $DO_STEP[$step_id];
|
|
my $output = &get_default_file(&deconstruct_name($step));
|
|
|
|
my @INPUT;
|
|
if (defined($USES_INPUT{$step_id})) {
|
|
for(my $i=0; $i<scalar @{$USES_INPUT{$step_id}}; $i++) {
|
|
# get name of input file needed
|
|
my $in_file = $USES_INPUT{$step_id}[$i];
|
|
|
|
# if not directly specified, find step that produces this file.
|
|
# note that if the previous step is passed than the grandparent's
|
|
# outfile is used (done by &get_specified_or_default_file)
|
|
my $prev_step = "";
|
|
# print "\tlooking up in_file $in_file\n";
|
|
foreach my $parent (@{$DEPENDENCY[$step_id]}) {
|
|
my ($parent_module,$parent_set,$parent_step)
|
|
= &deconstruct_name($DO_STEP[$parent]);
|
|
my $parent_file
|
|
= &construct_name($parent_module,$parent_set,
|
|
$STEP_OUT{&defined_step($DO_STEP[$parent])});
|
|
if ($in_file eq $parent_file) {
|
|
$prev_step = $DO_STEP[$parent];
|
|
}
|
|
}
|
|
# print "\t\tfrom previous step: $prev_step ($in_file)\n";
|
|
if ($prev_step eq "" && !defined($CONFIG{$in_file})) {
|
|
# undefined (ignored previous step)
|
|
# print "ignored previous step to generate $USES_INPUT{$step_id}[$i]\n";
|
|
push @INPUT,"";
|
|
next;
|
|
}
|
|
|
|
# get the actual file name
|
|
push @INPUT,&get_specified_or_default_file(&deconstruct_name($in_file),
|
|
&deconstruct_name($prev_step));
|
|
}
|
|
}
|
|
return ($output,@INPUT);
|
|
}
|
|
|
|
sub define_template {
|
|
my ($step_id) = @_;
|
|
|
|
my $step = $DO_STEP[$step_id];
|
|
print "building sh file for $step\n" if $VERBOSE;
|
|
my $defined_step = &defined_step($step);
|
|
return 0 unless (defined($TEMPLATE {$defined_step}) ||
|
|
defined($TEMPLATE_IF{$defined_step}));
|
|
|
|
my $parallelizer = &get("GENERAL:generic-parallelizer");
|
|
my $dir = &check_and_get("GENERAL:working-dir");
|
|
|
|
my ($module,$set,$stepname) = &deconstruct_name($step);
|
|
|
|
my $multiref = undef;
|
|
if ($MULTIREF{$defined_step} && # step needs to be run differently if multiple ref
|
|
&backoff_and_get(&extend_local_name($module,$set,"multiref"))) { # there are multiple ref
|
|
$multiref = $MULTIREF{$defined_step};
|
|
}
|
|
|
|
my ($output,@INPUT) = &get_output_and_input($step_id);
|
|
|
|
my $cmd;
|
|
if (defined($TEMPLATE{$defined_step})) {
|
|
$cmd = $TEMPLATE{$defined_step};
|
|
}
|
|
else {
|
|
foreach my $template_if (@{$TEMPLATE_IF{$defined_step}}) {
|
|
my ($command,$in,$out,@EXTRA) = @{$template_if};
|
|
my $extra = join(" ",@EXTRA);
|
|
|
|
if (&backoff_and_get(&extend_local_name($module,$set,$command))) {
|
|
$cmd .= "\$$command < $in > $out $extra\n";
|
|
}
|
|
else {
|
|
$cmd .= "ln -s $in $out\n";
|
|
}
|
|
}
|
|
}
|
|
|
|
if ($parallelizer && defined($PARALLELIZE{$defined_step}) &&
|
|
((&get("$module:jobs") && $CLUSTER) ||
|
|
(&get("$module:cores") && $MULTICORE))) {
|
|
my $new_cmd;
|
|
my $i=0;
|
|
foreach my $single_cmd (split(/\n/,$cmd)) {
|
|
if ($single_cmd =~ /^ln /) {
|
|
$new_cmd .= $single_cmd."\n";
|
|
}
|
|
elsif ($single_cmd =~ /^.+$/) {
|
|
# find IN and OUT files
|
|
$single_cmd =~ /(EMS_IN_EMS\S*)/
|
|
|| die("ERROR: could not find EMS_IN_EMS in $single_cmd");
|
|
my $in = $1;
|
|
$single_cmd =~ /(EMS_OUT_EMS\S*)/
|
|
|| die("ERROR: could not find OUT in $single_cmd");
|
|
my $out = $1;
|
|
# replace IN and OUT with %s
|
|
$single_cmd =~ s/EMS_IN_EMS\S*/\%s/;
|
|
$single_cmd =~ s/EMS_OUT_EMS\S*/\%s/;
|
|
$single_cmd =~ s/EMS_SLASH_OUT_EMS\S*/\%s/;
|
|
# build tmp
|
|
my $tmp_dir = $module;
|
|
$tmp_dir =~ tr/A-Z/a-z/;
|
|
$tmp_dir .= "/tmp.$set.$stepname.$VERSION-".($i++);
|
|
if ($CLUSTER) {
|
|
my $qflags = "";
|
|
my $qsub_args = &get_qsub_args($DO_STEP[$step_id]);
|
|
$qflags="--queue-flags \"$qsub_args\"" if ($CLUSTER && $qsub_args);
|
|
$new_cmd .= "$parallelizer $qflags -in $in -out $out -cmd '$single_cmd' -jobs ".&get("$module:jobs")." -tmpdir $dir/$tmp_dir\n";
|
|
}
|
|
if ($MULTICORE) {
|
|
$new_cmd .= "$parallelizer -in $in -out $out -cmd '$single_cmd' -cores ".&get("$module:cores")." -tmpdir $dir/$tmp_dir\n";
|
|
}
|
|
}
|
|
}
|
|
|
|
$cmd = $new_cmd;
|
|
$QSUB_STEP{$step_id}++;
|
|
}
|
|
|
|
# command to be run on multiple reference translations
|
|
if (defined($multiref)) {
|
|
$cmd =~ s/^(.*)EMS_IN_EMS (.+)EMS_OUT_EMS(.*)$/$multiref '$1 mref-input-file $2 mref-output-file $3' EMS_IN_EMS EMS_OUT_EMS/;
|
|
$cmd =~ s/^(.+)EMS_OUT_EMS(.+)EMS_IN_EMS (.*)$/$multiref '$1 mref-output-file $2 mref-input-file $3' EMS_IN_EMS EMS_OUT_EMS/;
|
|
}
|
|
|
|
# input is array, but just specified as IN
|
|
if ($cmd !~ /EMS_IN1_EMS/ && (scalar @INPUT) > 1 ) {
|
|
my $in = join(" ",@INPUT);
|
|
$cmd =~ s/EMS_IN_EMS/$in/;
|
|
}
|
|
# input is defined as IN or IN0, IN1, IN2
|
|
else {
|
|
if ($cmd =~ /EMS_IN\d*_EMS/ && scalar(@INPUT) == 0) {
|
|
die("ERROR: Step $step requires input from prior steps, but none defined.");
|
|
}
|
|
$cmd =~ s/EMS_IN(\d)_EMS/$INPUT[$1]/g;
|
|
$cmd =~ s/EMS_IN_EMS/$INPUT[0]/g;
|
|
}
|
|
$cmd =~ s/EMS_OUT_EMS/$output/g;
|
|
if (defined($STEP_TMPNAME{"$module:$stepname"})) {
|
|
my $tmp = $dir."/".$STEP_TMPNAME{"$module:$stepname"}.".$VERSION";
|
|
$cmd =~ s/EMS_TMP_EMS/$tmp/g;
|
|
}
|
|
$cmd =~ s/VERSION/$VERSION/g;
|
|
print "\tcmd is $cmd\n" if $VERBOSE;
|
|
|
|
# replace variables
|
|
while ($cmd =~ /^([\S\s]*)\$(\??)\{([^\s\/\"\']+)\}([\S\s]*)$/ ||
|
|
$cmd =~ /^([\S\s]*)\$(\??)([^\s\/\"\']+)([\S\s]*)$/) {
|
|
my ($pre,$optional,$variable,$post) = ($1,$2,$3,$4);
|
|
my $value;
|
|
if ($optional eq '?') {
|
|
$value = &backoff_and_get(&extend_local_name($module,$set,$variable));
|
|
$value = "" unless $value;
|
|
}
|
|
else {
|
|
$value = &check_backoff_and_get(&extend_local_name($module,$set,$variable));
|
|
}
|
|
$cmd = $pre.$value.$post;
|
|
}
|
|
|
|
# deal with pipelined commands
|
|
$cmd =~ s/\|(.*)(\<\s*\S+) /$2 \| $1 /g;
|
|
|
|
# deal with gzipped input
|
|
my $c = "";
|
|
foreach my $cmd (split(/[\n\r]+/,$cmd)) {
|
|
if ($cmd =~ /\<\s*(\S+) / && ! -e $1 && -e "$1.gz") {
|
|
$cmd =~ s/([^\n\r]+)\s*\<\s*(\S+) /zcat $2.gz \| $1 /;
|
|
}
|
|
else {
|
|
$cmd =~ s/([^\n\r]+)\s*\<\s*(\S+\.gz)/zcat $2 \| $1/;
|
|
}
|
|
$c .= $cmd."\n";
|
|
}
|
|
$cmd = $c;
|
|
|
|
# create directory for output
|
|
if ($output =~ /\//) { # ... but why would it not?
|
|
my $out_dir = $output;
|
|
$out_dir =~ s/^(.+)\/[^\/]+$/$1/;
|
|
$cmd = "mkdir -p $out_dir\n$cmd";
|
|
}
|
|
|
|
&create_step($step_id,$cmd);
|
|
return 1;
|
|
}
|
|
|
|
### SUBS for defining steps
|
|
|
|
sub create_step {
|
|
my ($step_id,$cmd) = @_;
|
|
my ($module,$set,$step) = &deconstruct_name($DO_STEP[$step_id]);
|
|
my $file = &versionize(&step_file2($module,$set,$step));
|
|
my $dir = &check_and_get("GENERAL:working-dir");
|
|
my $subdir = $module;
|
|
$subdir =~ tr/A-Z/a-z/;
|
|
$subdir = "evaluation" if $subdir eq "reporting";
|
|
$subdir = "lm" if $subdir eq "interpolated-lm";
|
|
open(STEP,">$file") or die "Cannot open: $!";
|
|
print STEP "#!/bin/bash\n\n";
|
|
print STEP "PATH=\"".$ENV{"PATH"}."\"\n";
|
|
print STEP "cd $dir\n";
|
|
print STEP "echo 'starting at '`date`' on '`hostname`\n";
|
|
print STEP "mkdir -p $dir/$subdir\n\n";
|
|
print STEP "$cmd\n\n";
|
|
print STEP "echo 'finished at '`date`\n";
|
|
print STEP "touch $file.DONE\n";
|
|
close(STEP);
|
|
}
|
|
|
|
sub get {
|
|
return &check_and_get($_[0],"allow_undef");
|
|
}
|
|
|
|
sub check_and_get {
|
|
my ($parameter,$allow_undef) = @_;
|
|
if (!defined($CONFIG{$parameter})) {
|
|
return if $allow_undef;
|
|
print STDERR "ERROR: you need to define $parameter\n";
|
|
exit;
|
|
}
|
|
return $CONFIG{$parameter}[0];
|
|
}
|
|
|
|
sub backoff_and_get {
|
|
return &check_backoff_and_get($_[0],"allow_undef");
|
|
}
|
|
|
|
sub check_backoff_and_get {
|
|
my $VALUE = &check_backoff_and_get_array(@_);
|
|
return ${$VALUE}[0] if $VALUE;
|
|
}
|
|
|
|
sub backoff_and_get_array {
|
|
return &check_backoff_and_get_array($_[0],"allow_undef");
|
|
}
|
|
|
|
sub check_backoff_and_get_array {
|
|
my ($parameter,$allow_undef) = @_;
|
|
return $CONFIG{$parameter} if defined($CONFIG{$parameter});
|
|
|
|
# remove set -> find setting for module
|
|
$parameter =~ s/:[^:]+:/:/;
|
|
return $CONFIG{$parameter} if defined($CONFIG{$parameter});
|
|
|
|
# remove step (if exists)
|
|
if ($parameter =~ /:[^:]+:/) {
|
|
$parameter =~ s/:[^:]+:/:/;
|
|
return $CONFIG{$parameter} if defined($CONFIG{$parameter});
|
|
}
|
|
|
|
# remove model -> find global setting
|
|
$parameter =~ s/^[^:]+:/GENERAL:/;
|
|
return $CONFIG{$parameter} if defined($CONFIG{$parameter});
|
|
|
|
return if $allow_undef;
|
|
print STDERR "ERROR: you need to define $parameter\n";
|
|
exit;
|
|
}
|
|
|
|
# the following two functions deal with getting information about
|
|
# files that are passed between steps. this are either specified
|
|
# in the meta file (default) or in the configuration file (here called
|
|
# 'specified', in the step management referred to as 'given').
|
|
|
|
sub get_specified_or_default_file {
|
|
my ($specified_module,$specified_set,$specified_parameter,
|
|
$default_module, $default_set, $default_step) = @_;
|
|
my $specified =
|
|
&construct_name($specified_module,$specified_set,$specified_parameter);
|
|
if (defined($CONFIG{$specified})) {
|
|
print "\t\texpanding $CONFIG{$specified}[0]\n" if $VERBOSE;
|
|
return &long_file_name($CONFIG{$specified}[0],$default_module,$default_set);
|
|
}
|
|
return &get_default_file($default_module, $default_set, $default_step);
|
|
}
|
|
|
|
sub get_tmp_file {
|
|
my ($module,$set,$step,$version) = @_;
|
|
$version = $VERSION unless $version;
|
|
my $tmp_file = $STEP_TMPNAME{"$module:$step"};
|
|
if ($set) {
|
|
$tmp_file =~ s/^(.+\/)([^\/]+)$/$1$set.$2/g;
|
|
}
|
|
$tmp_file = &versionize(&long_file_name($tmp_file,$module,$set), $version);
|
|
return $tmp_file;
|
|
}
|
|
|
|
sub get_default_file {
|
|
my ($default_module, $default_set, $default_step) = @_;
|
|
#print "\tget_default_file($default_module, $default_set, $default_step)\n";
|
|
|
|
# get step name
|
|
my $step = &construct_name($default_module,$default_set,$default_step);
|
|
#print "\t\tstep is $step\n";
|
|
|
|
# earlier step, if this step is passed
|
|
my $i = $STEP_LOOKUP{$step};
|
|
#print "\t\tcan't lookup $step -> $i!\n" unless $i;
|
|
while (defined($PASS{$i})) {
|
|
if (scalar @{$DEPENDENCY[$i]} == 0) {
|
|
#print "\t\tpassing to given\n";
|
|
my $out = $STEP_IN{&defined_step($step)}[0];
|
|
my ($module,$set) = &deconstruct_name($step);
|
|
foreach my $out_option (split(/=OR=/,$out)) {
|
|
#print "\t\t$out_option -> ".&construct_name($module,$set,$out_option)."\n";
|
|
my $name = &construct_name($module,$set,$out);
|
|
return &get($name) if &get($name);
|
|
}
|
|
foreach my $out_option (split(/=OR=/,$out)) {
|
|
my $name = &construct_name($module,$set,$out_option);
|
|
return &backoff_and_get($name) if &backoff_and_get($name);
|
|
}
|
|
die("something is wrong with $out\n");
|
|
}
|
|
#print "\t\tpassing $step\n";
|
|
$i = $DEPENDENCY[$i][0];
|
|
$step = $DO_STEP[$i];
|
|
#print "\t\tbacking off to $step\n";
|
|
($default_module,$default_set,$default_step) = &deconstruct_name($step);
|
|
}
|
|
|
|
# get file name
|
|
my $default = $STEP_OUTNAME{&defined_step($step)};
|
|
# print "\t\tdefined_step is ".&defined_step($step)."\n";
|
|
die("no specified default name for $step") unless $default;
|
|
|
|
if ($default_set) {
|
|
$default =~ s/^(.+\/)([^\/]+)$/$1$default_set.$2/g;
|
|
}
|
|
|
|
# if from a step that is redone, get version number
|
|
my $version = 0;
|
|
$version = $RE_USE[$STEP_LOOKUP{$step}] if defined($STEP_LOOKUP{$step}) && $#RE_USE >= $STEP_LOOKUP{$step};
|
|
$version = "*" if $version > 1e6; # any if re-use checking
|
|
$version = $VERSION unless $version; # current version if no re-use
|
|
|
|
return &versionize(&long_file_name($default,$default_module,$default_set),
|
|
$version);
|
|
}
|
|
|
|
sub long_file_name {
|
|
my ($file,$module,$set) = @_;
|
|
return $file if $file =~ /^\// || $file =~ / \//;
|
|
|
|
if ($file !~ /\//) {
|
|
my $dir = $module;
|
|
$dir =~ tr/A-Z/a-z/;
|
|
$file = "$dir/$file";
|
|
}
|
|
|
|
my $module_working_dir_parameter =
|
|
$module . ($set ne "" ? ":$set" : "") . ":working-dir";
|
|
|
|
if (defined($CONFIG{$module_working_dir_parameter})) {
|
|
return $CONFIG{$module_working_dir_parameter}[0]."/".$file;
|
|
}
|
|
return &check_and_get("GENERAL:working-dir")."/".$file;
|
|
}
|
|
|
|
sub compute_version_number {
|
|
my $dir = &check_and_get("GENERAL:working-dir");
|
|
$VERSION = 1;
|
|
return unless -e $dir;
|
|
open(LS,"find $dir/steps -maxdepth 1 -follow |");
|
|
while(<LS>) {
|
|
s/.+\/([^\/]+)$/$1/; # ignore path
|
|
if ( /^(\d+)$/ ) {
|
|
if ($1 >= $VERSION) {
|
|
$VERSION = $1 + 1;
|
|
}
|
|
}
|
|
}
|
|
close(LS);
|
|
}
|
|
|
|
sub steps_file {
|
|
my ($file,$run) = @_;
|
|
return "steps/$run/$file";
|
|
}
|