mirror of
https://github.com/moses-smt/mosesdecoder.git
synced 2024-12-26 21:42:19 +03:00
checking in the current version of clone_moses_model
working on a single scripts directory git-svn-id: https://mosesdecoder.svn.sourceforge.net/svnroot/mosesdecoder/trunk@416 1f5c12ca-751b-0410-a591-d2e778427230
This commit is contained in:
bojar
parent
66aeb9fe27
commit
1c2cd47881
@ -1,18 +1,46 @@
|
||||
# This makefile is here to simplify the automatic releases (and tests!!!)
|
||||
# of the scripts
|
||||
|
||||
|
||||
TS?=$(shell date '+%Y%m%d-%H%M')
|
||||
RELEASEDIR=/export/ws06osmt/bin/scripts-$(TS)
|
||||
DS?=$(shell date '+%Y%m%d')
|
||||
|
||||
### "MAIN" scripts are scripts that have a Philipp-like name, too
|
||||
## That is for each script (listed below in MAIN_SCRIPTS),
|
||||
## we create a date-stamped version in MAIN_SCRIPTS_TARGET_DIR
|
||||
|
||||
# MAIN_SCRIPTS_TARGET_DIR=/export/ws06osmt/bin
|
||||
MAIN_SCRIPTS_TARGET_DIR=$(shell echo `pwd`/temp)
|
||||
|
||||
# RELEASEDIR=/export/ws06osmt/bin/scripts-$(TS)
|
||||
RELEASEDIR=$(shell echo `pwd`/temp)
|
||||
|
||||
|
||||
VALID_TRAINING_SCRIPTS_NAMES=filter-model-given-input.pl mert-moses.pl train-factored-phrase-model.perl
|
||||
|
||||
|
||||
MAIN_TRAINING_SCRIPTS_NAMES=filter-model-given-input.pl mert-moses.pl train-factored-phrase-model.perl
|
||||
# Make trick to add directory name to all of them:
|
||||
VALID_TRAINING_SCRIPTS=$(VALID_TRAINING_SCRIPTS_NAMES:%=training/%)
|
||||
MAIN_TRAINING_SCRIPTS=$(MAIN_TRAINING_SCRIPTS_NAMES:%=training/%)
|
||||
|
||||
# the list of all scripts that should be released
|
||||
VALID_SCRIPTS= $(VALID_TRAINING_SCRIPTS)
|
||||
MAIN_SCRIPTS= $(MAIN_TRAINING_SCRIPTS)
|
||||
|
||||
release:
|
||||
if [ -e $(RELEASEDIR) ]; then echo "Targetdir exists! Not touching it! $(RELEASEDIR)"; exit 1; fi
|
||||
mkdir -p $(RELEASEDIR)
|
||||
cp $(VALID_SCRIPTS) $(RELEASEDIR)
|
||||
rsync -r --files-from ./released-files . $(RELEASEDIR)/
|
||||
## Now adding JHU-specific stuff
|
||||
mkdir -p $(RELEASEDIR)/extra/
|
||||
cp -r /export/ws06osmt/bin/cmert-0.5 $(RELEASEDIR)/extra/
|
||||
## And for each script, create/rewrite the daily release
|
||||
for s in $(MAIN_SCRIPTS); do \
|
||||
bn=`basename $$s`; \
|
||||
echo "export SCRIPTS_ROOTDIR=$(RELEASEDIR); $(RELEASEDIR)/$$s" > $(MAIN_SCRIPTS_TARGET_DIR)/$$bn-$(DS) || exit 1; \
|
||||
chmod 775 $(MAIN_SCRIPTS_TARGET_DIR)/$$bn-$(DS); \
|
||||
done
|
||||
## URGE yourself to:
|
||||
@echo " export SCRIPTS_ROOTDIR=$(RELEASEDIR)"
|
||||
## Release succeeded, tag the CVS
|
||||
cvs tag SCRIPTS-RELEASE-$(TS)
|
||||
|
||||
|
||||
|
||||
@ -5,4 +5,12 @@ This directory should contain all multi-purpose scripts for:
|
||||
- training ... training moses (including BLEU evaluation needed for MERT)
|
||||
- analysis ... analyzing MT output (for human analysis)
|
||||
|
||||
- lib ... perl modules used by various scripts
|
||||
|
||||
|
||||
The Makefile then takes care of proper 'release' from your CVS directory to
|
||||
the shared directories.
|
||||
|
||||
The released scripts should remain in the *same directory structure*.
|
||||
|
||||
|
||||
|
||||
75
scripts/training/clone_moses_model.pl
Executable file
75
scripts/training/clone_moses_model.pl
Executable file
@ -0,0 +1,75 @@
|
||||
#!/usr/bin/perl
|
||||
# given a moses.ini file, creates a wiseln of it and all the included bits
|
||||
# in the current directory
|
||||
|
||||
# relies on wiseln, a wise variant of linking. You might just use ln -s instead.
|
||||
|
||||
my $ini = shift;
|
||||
die "usage!" if !defined $ini;
|
||||
|
||||
my %cnt; # count files per section
|
||||
open INI, $ini or die "Can't read $ini";
|
||||
open OUT, ">moses.ini" or die "Can't write ./moses.ini";
|
||||
while (<INI>) {
|
||||
if (/^\[([^\]]*)\]\s*$/) {
|
||||
$section = $1;
|
||||
}
|
||||
if (/^[0-9]/) {
|
||||
if ($section eq "ttable-file" || $section eq "lmodel-file") {
|
||||
chomp;
|
||||
my ($a, $b, $c, $fn) = split / /;
|
||||
$cnt{$section}++;
|
||||
my $suffix = ($fn =~ /\.gz$/ ? ".gz" : "");
|
||||
$fn = ensure_relative_to_origin($fn, $ini);
|
||||
safesystem("wiseln $fn ./$section.$cnt{$section}$suffix") or die;
|
||||
$_ = "$a $b $c ./$section.$cnt{$section}$suffix\n";
|
||||
}
|
||||
if ($section eq "generation-file") {
|
||||
chomp;
|
||||
my ($a, $b, $fn) = split / /;
|
||||
$cnt{$section}++;
|
||||
my $suffix = ($fn =~ /\.gz$/ ? ".gz" : "");
|
||||
$fn = ensure_relative_to_origin($fn, $ini);
|
||||
safesystem("wiseln $fn ./$section.$cnt{$section}$suffix") or die;
|
||||
$_ = "$a $b ./$section.$cnt{$section}$suffix\n";
|
||||
}
|
||||
if ($section eq "distortion-file") {
|
||||
chomp;
|
||||
my $fn = $_;
|
||||
$cnt{$section}++;
|
||||
my $suffix = ($fn =~ /\.gz$/ ? ".gz" : "");
|
||||
$fn = ensure_relative_to_origin($fn, $ini);
|
||||
safesystem("wiseln $fn ./$section.$cnt{$section}$suffix") or die;
|
||||
$_ = "./$section.$cnt{$section}$suffix\n";
|
||||
}
|
||||
}
|
||||
print OUT $_;
|
||||
}
|
||||
close INI;
|
||||
close OUT;
|
||||
|
||||
sub safesystem {
|
||||
print STDERR "Executing: @_\n";
|
||||
system(@_);
|
||||
if ($? == -1) {
|
||||
print STDERR "Failed to execute: @_\n $!\n";
|
||||
exit(1);
|
||||
}
|
||||
elsif ($? & 127) {
|
||||
printf STDERR "Execution of: @_\n died with signal %d, %s coredump\n",
|
||||
($? & 127), ($? & 128) ? 'with' : 'without';
|
||||
}
|
||||
else {
|
||||
my $exitcode = $? >> 8;
|
||||
print STDERR "Exit code: $exitcode\n" if $exitcode;
|
||||
return ! $exitcode;
|
||||
}
|
||||
}
|
||||
|
||||
sub ensure_relative_to_origin {
|
||||
my $target = shift;
|
||||
my $originfile = shift;
|
||||
return $target if $target =~ /^\/|^~/; # the target path is absolute already
|
||||
$originfile =~ s/[^\/]*$//;
|
||||
return $originfile."/".$target;
|
||||
}
|
||||
@ -9,6 +9,7 @@
|
||||
|
||||
# Revision history
|
||||
|
||||
# 31 Jul 2006 adding default paths
|
||||
# 29 Jul 2006 run-filter, score-nbest and mert run on the queue (Nicola; Ondrej had to type it in again)
|
||||
# 28 Jul 2006 attempt at foolproof usage, strong checking of input validity, merged the parallel and nonparallel version (Ondrej Bojar)
|
||||
# 27 Jul 2006 adding the safesystem() function to handle with process failure
|
||||
@ -85,7 +86,7 @@ my $___START_STEP = undef; # which iteration step to start with
|
||||
# Use "--average" to use average reference length
|
||||
my $___AVERAGE = 0;
|
||||
|
||||
my $bindir = undef; # path to all tools (overriden by specific options)
|
||||
my $SCRIPTS_ROOTDIR = undef; # path to all tools (overriden by specific options)
|
||||
my $cmertdir = undef; # path to cmert directory
|
||||
my $pythonpath = undef; # path to python libraries needed by cmert
|
||||
my $filtercmd = undef; # path to filter-model-given-input.pl
|
||||
@ -111,7 +112,7 @@ GetOptions(
|
||||
"average" => \$___AVERAGE,
|
||||
"help" => \$usage,
|
||||
"verbose" => \$verbose,
|
||||
"bindir=s" => \$bindir,
|
||||
"roodir=s" => \$SCRIPTS_ROOTDIR,
|
||||
"cmertdir=s" => \$cmertdir,
|
||||
"pythonpath=s" => \$pythonpath,
|
||||
"filtercmd=s" => \$filtercmd, # allow to override the default location
|
||||
@ -144,7 +145,7 @@ Options:
|
||||
--average ... Use either average or shortest (default) reference
|
||||
length as effective reference length
|
||||
--filtercmd=STRING ... path to filter-model-given-input.pl
|
||||
--bindir=STRING ... where do helpers reside (if not given explicitly)
|
||||
--roodir=STRING ... where do helpers reside (if not given explicitly)
|
||||
--cmertdir=STRING ... where is cmert installed
|
||||
--pythonpath=STRING ... where is python executable
|
||||
--scorenbestcmd=STRING ... path to score-nbest.py
|
||||
@ -155,15 +156,25 @@ Options:
|
||||
# Check validity of input parameters
|
||||
|
||||
|
||||
$bindir = $ENV{"MOSESBIN"} if !defined $bindir;
|
||||
|
||||
|
||||
if (!defined $SCRIPTS_ROOTDIR) {
|
||||
$SCRIPTS_ROOTDIR = $ENV{"SCRIPTS_ROOTDIR"};
|
||||
die "Please set SCRIPTS_ROOTDIR or specify --rootdir" if !defined $SCRIPTS_ROOTDIR;
|
||||
}
|
||||
|
||||
print STDERR "Using SCRIPTS_ROOTDIR: $SCRIPTS_ROOTDIR\n";
|
||||
|
||||
|
||||
|
||||
|
||||
# path of script for filtering phrase tables and running the decoder
|
||||
$filtercmd="$bindir/filter-model-given-input.pl" if !defined $filtercmd;
|
||||
$filtercmd="$SCRIPTS_ROOTDIR/training/filter-model-given-input.pl" if !defined $filtercmd;
|
||||
|
||||
$qsubwrapper="$bindir/qsub-wrapper.pl" if !defined $qsubwrapper;
|
||||
$qsubwrapper="$SCRIPTS_ROOTDIR/training/qsub-wrapper.pl" if !defined $qsubwrapper;
|
||||
|
||||
|
||||
$cmertdir = "$bindir/cmert-0.5" if !defined $cmertdir;
|
||||
$cmertdir = "$SCRIPTS_ROOTDIR/extra/cmert-0.5" if !defined $cmertdir;
|
||||
my $cmertcmd="$cmertdir/mert";
|
||||
|
||||
$SCORENBESTCMD = "$cmertdir/score-nbest.py" if ! defined $SCORENBESTCMD;
|
||||
|
||||
Loading…
Reference in New Issue
Block a user