mirror of
https://github.com/moses-smt/mosesdecoder.git
synced 2024-08-16 23:10:31 +03:00
Support for Urdu in sentence splitter
This commit is contained in:
parent
158d252389
commit
7ad5ffa0c0
@ -165,6 +165,20 @@ sub preprocess {
|
||||
}{$1\n$2}gx;
|
||||
}
|
||||
|
||||
# Urdu support
|
||||
# https://en.wikipedia.org/wiki/Urdu_alphabet#Encoding_Urdu_in_Unicode
|
||||
if ($language eq 'ur') {
|
||||
$text =~ s{
|
||||
( (?: [\.\?!\x{06d4}] | \.\.+ )
|
||||
[\'\"\x{201e}\x{bb}\(\[\¿\¡\p{IsPf}]*
|
||||
)
|
||||
\s+
|
||||
( [\'\"\x{201e}\x{bb}\(\[\¿\¡\p{IsPi}]*
|
||||
[\x{0600}-\x{06ff}]
|
||||
)
|
||||
}{$1\n$2}gx;
|
||||
}
|
||||
|
||||
# Special punctuation cases are covered. Check all remaining periods.
|
||||
my $word;
|
||||
my $i;
|
||||
|
Loading…
Reference in New Issue
Block a user