moses-smt/mosesdecoder
1
1
Fork 0
mirror of https://github.com/moses-smt/mosesdecoder.git synced 2024-08-16 15:00:33 +03:00
Code Issues Packages Projects Releases Wiki Activity

adding rules for Catalan

Browse Source 
special characters within words and contractions closer to French than to English
This commit is contained in:
Cristina España i Bonet 2020-07-31 15:22:47 +02:00 committed by GitHub
parent 47915b561f
commit 8d78dae634
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 8 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
scripts/tokenizer/tokenizer.perl
View File

@ -265,6 +265,13 @@ sub tokenize
# if a colon is not immediately followed by lower-case characters, separate it out anyway
$text =~ s/(:)(?=$|[^\p{Ll}])/ $1 /g;
}
elsif (($language eq "ca")) {
# in Catalan, the middle dot can be used inside words:
# il<69>lusio
$text =~ s/([^\p{IsAlnum}\s\.\·\'\`\,\-])/ $1 /g;
# if a middot is not immediately followed by lower-case characters, separate it out anyway
$text =~ s/(·)(?=$|[^\p{Ll}])/ $1 /g;
}
else {
$text =~ s/([^\p{IsAlnum}\s\.\'\`\,\-])/ $1 /g;
}
@ -317,7 +324,7 @@ sub tokenize
#special case for "1990's"
$text =~ s/([\p{IsN}])[']([s])/$1 '$2/g;
}
elsif (($language eq "fr") or ($language eq "it") or ($language eq "ga"))
elsif (($language eq "fr") or ($language eq "it") or ($language eq "ga") or ($language eq "ca"))
{
#split contractions left
$text =~ s/([^\p{IsAlpha}])[']([^\p{IsAlpha}])/$1 ' $2/g;