mirror of
https://github.com/moses-smt/mosesdecoder.git
synced 2024-09-11 11:25:40 +03:00
tokenizer.perl: split final dots unconditionally
Allow tokenization of non-breaking prefixes at end of sentences. This should be a fair compromise in many cases to construct a cleaner vocabulary. EN-old: So am I. EN-new: So am I . DE-old: ... schwer wie ein iPhone 5. DE-new: ... schwer wie ein iPhone 5 . FR-old: Des gens admirent une œuvre d' art. FR-new: Des gens admirent une œuvre d' art . CS-old: Dvě děti, které běží bez bot. CS-new: Dvě děti, které běží bez bot .
This commit is contained in:
parent
d2b558728f
commit
9fc964da7f
@ -346,10 +346,14 @@ sub tokenize
|
||||
if ( $word =~ /^(\S+)\.$/)
|
||||
{
|
||||
my $pre = $1;
|
||||
if (($pre =~ /\./ && $pre =~ /\p{IsAlpha}/) || ($NONBREAKING_PREFIX{$pre} && $NONBREAKING_PREFIX{$pre}==1) || ($i<scalar(@words)-1 && ($words[$i+1] =~ /^[\p{IsLower}]/)))
|
||||
if ($i == scalar(@words)-1) {
|
||||
# split last words independently as they are unlikely to be non-breaking prefixes
|
||||
$word = $pre." .";
|
||||
}
|
||||
elsif (($pre =~ /\./ && $pre =~ /\p{IsAlpha}/) || ($NONBREAKING_PREFIX{$pre} && $NONBREAKING_PREFIX{$pre}==1) || ($i<scalar(@words)-1 && ($words[$i+1] =~ /^[\p{IsLower}]/)))
|
||||
{
|
||||
#no change
|
||||
}
|
||||
}
|
||||
elsif (($NONBREAKING_PREFIX{$pre} && $NONBREAKING_PREFIX{$pre}==2) && ($i<scalar(@words)-1 && ($words[$i+1] =~ /^[0-9]+/)))
|
||||
{
|
||||
#no change
|
||||
|
Loading…
Reference in New Issue
Block a user