moses-smt/mosesdecoder
1
1
Fork 0
mirror of https://github.com/moses-smt/mosesdecoder.git synced 2024-09-11 11:25:40 +03:00
Code Issues Packages Projects Releases Wiki Activity

ignore words where there is nothing to case

Browse Source
This commit is contained in:
Ondrej Bojar 2017-04-07 17:28:13 +02:00
parent 1d65006190
commit d9faf8f901
1 changed files with 6 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
scripts/recaser/train-truecaser.perl
View File

@ -44,6 +44,12 @@ while(<CORPUS>) {
$firstWordOfSentence = 1;
}
if ($currentWord !~ /[\p{Ll}\p{Lu}\p{Lt}]/) {
# skip words with nothing to case
$firstWordOfSentence = 0;
next;
}
my $currentWordWeight = 0;
if (! $firstWordOfSentence) {
$currentWordWeight = 1;