moses-smt/mosesdecoder
1
1
Fork 0
mirror of https://github.com/moses-smt/mosesdecoder.git synced 2024-09-11 11:25:40 +03:00
Code Issues Packages Projects Releases Wiki Activity

another madamira wrapper. Just uses the tokenized file it outputs

Browse Source
This commit is contained in:
Hieu Hoang 2015-04-30 15:35:15 +04:00
parent 15e4b16f49
commit a6d34a660d
1 changed files with 97 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

97
scripts/training/wrappers/madamira-tok.perl Executable file
View File

@ -0,0 +1,97 @@
#!/usr/bin/env perl
use warnings;
use strict;
use File::Temp qw/tempfile/;
use Getopt::Long "GetOptions";
use File::Basename;
use FindBin qw($RealBin);
use Cwd 'abs_path';
sub GetFactors;
my $TMPDIR = "tmp";
my $KEEP_TMP = 0;
my $MADA_DIR;
my $CONFIG;
my $SCHEME;
my $FACTORS_STR;
my @FACTORS;
GetOptions(
"tmpdir=s" => \$TMPDIR,
"keep-tmp" => \$KEEP_TMP,
"mada-dir=s" => \$MADA_DIR,
"factors=s" => \$FACTORS_STR,
"config=s" => \$CONFIG,
"scheme=s" => \$SCHEME
) or die("ERROR: unknown options");
die("must have -scheme arg") unless defined($SCHEME);
if (!defined($CONFIG)) {
$CONFIG = "$MADA_DIR/samples/sampleConfigFile.xml";
}
$TMPDIR = abs_path($TMPDIR);
print STDERR "TMPDIR=$TMPDIR \n";
if (defined($FACTORS_STR)) {
@FACTORS = split(",", $FACTORS_STR);
}
#binmode(STDIN, ":utf8");
#binmode(STDOUT, ":utf8");
$TMPDIR = "$TMPDIR/madamira.$$";
`mkdir -p $TMPDIR`;
`mkdir -p $TMPDIR/split`;
`mkdir -p $TMPDIR/out`;
my $infile = "$TMPDIR/input";
print STDERR $infile."\n";
open(TMP,">$infile");
while(<STDIN>) {
print TMP $_;
}
close(TMP);
my $cmd;
# split input file
my $SPLIT_EXEC = `gsplit --help 2>/dev/null`;
if($SPLIT_EXEC) {
$SPLIT_EXEC = 'gsplit';
}
else {
$SPLIT_EXEC = 'split';
}
$cmd = "$SPLIT_EXEC -l 10000 -a 7 -d $TMPDIR/input $TMPDIR/split/x";
`$cmd`;
$cmd = "cd $MADA_DIR && parallel --jobs 4 java -Xmx2500m -Xms2500m -XX:NewRatio=3 -jar $MADA_DIR/MADAMIRA.jar -rawinput {} -rawoutdir $TMPDIR/out -rawconfig $CONFIG ::: $TMPDIR/split/x*";
print STDERR "Executing: $cmd\n";
`$cmd`;
$cmd = "cat $TMPDIR/out/x*.$SCHEME.tok > $infile.mada";
print STDERR "Executing: $cmd\n";
`$cmd`;
# get stuff out of mada output
open(MADA_OUT,"<$infile.mada");
#binmode(MADA_OUT, ":utf8");
while(my $line = <MADA_OUT>) {
chomp($line);
print "$line\n";
}
close (MADA_OUT);
if ($KEEP_TMP == 0) {
# `rm -rf $TMPDIR`;
}