moses-smt/mosesdecoder
1
1
Fork 0
mirror of https://github.com/moses-smt/mosesdecoder.git synced 2024-12-27 22:14:57 +03:00
Code Issues Packages Projects Releases Wiki Activity

parse-de-bitpar.perl: fix special char handling

Browse Source 
Unescape special characters in input to BitPar and then re-escape
in output.
This commit is contained in:
Phil Williams 2012-10-06 16:27:33 +01:00
parent 4fd980c386
commit b2b9751227
1 changed files with 10 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
scripts/training/wrappers/parse-de-bitpar.perl
View File

@ -9,6 +9,9 @@ use File::Temp qw/tempfile/;
my $BITPAR = "/exports/home/s0565741/work/bin/bitpar";
my $TMPDIR = "tmp";
my $SCRIPTS_ROOT_DIR = "$RealBin/../..";
my $DEESCAPE = "$SCRIPTS_ROOT_DIR/tokenizer/deescape-special-chars.perl";
my $DEBUG = 0;
my $BASIC = 0;
my $OLD_BITPAR = 0;
@ -27,11 +30,11 @@ my ($scriptname, $directories) = fileparse($0);
my ($TMP, $tmpfile) = tempfile("$scriptname-XXXXXXXXXX", DIR=>$TMPDIR, UNLINK=>1);
if ($OLD_BITPAR)
{
open(INPUT,"iconv -c -f UTF-8 -t iso-8859-1 |");
open(INPUT,"$DEESCAPE | iconv -c -f UTF-8 -t iso-8859-1 |");
}
else
{
open (INPUT,"cat |");
open (INPUT,"$DEESCAPE |");
}
while(<INPUT>)
{
@ -162,7 +165,12 @@ sub is_aux_label {
sub escape {
my ($text) = @_;
$text =~ s/&/&amp;/g;
$text =~ s/\|/&#124;/g;
$text =~ s/</&lt;/g;
$text =~ s/>/&gt;/g;
$text =~ s/'/&apos;/g;
$text =~ s/"/&quot;/g;
$text =~ s/\[/&#91;/g;
$text =~ s/\]/&#93;/g;
return $text;
}