Merge ../mosesdecoder into perf_moses2

This commit is contained in:
Hieu Hoang 2016-08-05 17:15:18 +01:00
commit bf4f6b3b90
2 changed files with 4 additions and 1 deletions

View File

@ -30,7 +30,7 @@ W
X
Y
Z
a
#a
b
c
d

View File

@ -284,6 +284,9 @@ sub tokenize
# will also space digit,letter or letter,digit forms (redundant with next section)
$text =~ s/([^\p{IsN}])[,]/$1 , /g;
$text =~ s/[,]([^\p{IsN}])/ , $1/g;
# separate "," after a number if it's the end of a sentence
$text =~ s/([\p{IsN}])[,]$/$1 ,/g;
# separate , pre and post number
#$text =~ s/([\p{IsN}])[,]([^\p{IsN}])/$1 , $2/g;