mirror of
https://github.com/moses-smt/mosesdecoder.git
synced 2024-09-11 11:25:40 +03:00
Support for Urdu in sentence splitter
This commit is contained in:
parent
158d252389
commit
7ad5ffa0c0
@ -165,6 +165,20 @@ sub preprocess {
|
|||||||
}{$1\n$2}gx;
|
}{$1\n$2}gx;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
# Urdu support
|
||||||
|
# https://en.wikipedia.org/wiki/Urdu_alphabet#Encoding_Urdu_in_Unicode
|
||||||
|
if ($language eq 'ur') {
|
||||||
|
$text =~ s{
|
||||||
|
( (?: [\.\?!\x{06d4}] | \.\.+ )
|
||||||
|
[\'\"\x{201e}\x{bb}\(\[\¿\¡\p{IsPf}]*
|
||||||
|
)
|
||||||
|
\s+
|
||||||
|
( [\'\"\x{201e}\x{bb}\(\[\¿\¡\p{IsPi}]*
|
||||||
|
[\x{0600}-\x{06ff}]
|
||||||
|
)
|
||||||
|
}{$1\n$2}gx;
|
||||||
|
}
|
||||||
|
|
||||||
# Special punctuation cases are covered. Check all remaining periods.
|
# Special punctuation cases are covered. Check all remaining periods.
|
||||||
my $word;
|
my $word;
|
||||||
my $i;
|
my $i;
|
||||||
|
Loading…
Reference in New Issue
Block a user