mirror of
https://github.com/moses-smt/mosesdecoder.git
synced 2024-09-11 11:25:40 +03:00
Merge ../mosesdecoder into perf_moses2
This commit is contained in:
commit
bf4f6b3b90
@ -30,7 +30,7 @@ W
|
||||
X
|
||||
Y
|
||||
Z
|
||||
a
|
||||
#a
|
||||
b
|
||||
c
|
||||
d
|
||||
|
@ -284,6 +284,9 @@ sub tokenize
|
||||
# will also space digit,letter or letter,digit forms (redundant with next section)
|
||||
$text =~ s/([^\p{IsN}])[,]/$1 , /g;
|
||||
$text =~ s/[,]([^\p{IsN}])/ , $1/g;
|
||||
|
||||
# separate "," after a number if it's the end of a sentence
|
||||
$text =~ s/([\p{IsN}])[,]$/$1 ,/g;
|
||||
|
||||
# separate , pre and post number
|
||||
#$text =~ s/([\p{IsN}])[,]([^\p{IsN}])/$1 , $2/g;
|
||||
|
Loading…
Reference in New Issue
Block a user