moses-smt/mosesdecoder
1
1
Fork 0
mirror of https://github.com/moses-smt/mosesdecoder.git synced 2024-09-20 23:58:15 +03:00
Code Issues Packages Projects Releases Wiki Activity

Option to disable buffering (from Tom Hoar)

Browse Source
This commit is contained in:
Barry Haddow 2012-01-18 08:55:12 +00:00
parent ddd3d97bcb
commit 07e084fc75
2 changed files with 9 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
scripts/tokenizer/detokenizer.perl
View File

@ -1,13 +1,10 @@
#!/usr/bin/perl -w
# $Id$
# $Id: detokenizer.perl 4134 2011-08-08 15:30:54Z bgottesman $
# Sample De-Tokenizer
# written by Josh Schroeder, based on code by Philipp Koehn
# further modifications by Ondrej Bojar
# This added by Herve Saint-Amand for compatibility with translate.cgi
$|++;
binmode(STDIN, ":utf8");
binmode(STDOUT, ":utf8");
use strict;
@ -20,6 +17,7 @@ my $UPPERCASE_SENT = 0;
while (@ARGV) {
$_ = shift;
/^-b$/ && ($| = 1, next);
/^-l$/ && ($language = shift, next);
/^-q$/ && ($QUIET = 1, next);
/^-h$/ && ($HELP = 1, next);
@ -31,6 +29,7 @@ if ($HELP) {
print "Options:\n";
print " -u ... uppercase the first char in the final sentence.\n";
print " -q ... don't report detokenizer revision.\n";
print " -b ... disable Perl buffering.\n";
exit;
}
@ -38,7 +37,7 @@ die "No built-in rules for language $language, claim en for default behaviour."
if $language !~ /^(cs|en|fr|it)$/;
if (!$QUIET) {
print STDERR "Detokenizer Version ".'$Revision$'."\n";
print STDERR "Detokenizer Version ".'$Revision: 4134 $'."\n";
print STDERR "Language: $language\n";
}

5
scripts/tokenizer/tokenizer.perl
View File

@ -23,6 +23,7 @@ my $AGGRESSIVE = 0;
while (@ARGV) {
$_ = shift;
/^-b$/ && ($| = 1, next);
/^-l$/ && ($language = shift, next);
/^-q$/ && ($QUIET = 1, next);
/^-h$/ && ($HELP = 1, next);
@ -31,6 +32,10 @@ while (@ARGV) {
if ($HELP) {
print "Usage ./tokenizer.perl (-l [en|de|...]) < textfile > tokenizedfile\n";
print "Options:\n";
print " -q ... quiet.\n";
print " -a ... aggressive hyphen splitting.\n";
print " -b ... disable Perl buffering.\n";
exit;
}
if (!$QUIET) {