moses-smt/mosesdecoder
1
1
Fork 0
mirror of https://github.com/moses-smt/mosesdecoder.git synced 2025-01-01 00:12:27 +03:00
Code Issues Packages Projects Releases Wiki Activity

Add a space before double punctuation signs in French

Browse Source 
git-svn-id: https://mosesdecoder.svn.sourceforge.net/svnroot/mosesdecoder/trunk@4016 1f5c12ca-751b-0410-a591-d2e778427230
This commit is contained in:
rafpayen 2011-06-16 17:24:25 +00:00
parent 85283f5bee
commit cdc4179ce1
1 changed files with 3 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
scripts/tokenizer/detokenizer.perl
View File

@ -78,6 +78,9 @@ sub detokenize {
$text = $text.$prependSpace.$words[$i];
$prependSpace = "";
} elsif ($words[$i] =~ /^[\,\.\?\!\:\;\\\%\}\]\)]+$/){
if (($language eq "fr") && ($words[$i] =~ /^[\?\!\:\;\\\%]$/)) {
#these punctuations are prefixed with a non-breakable space in french
$text .= " "; }
#perform left shift on punctuation items
$text=$text.$words[$i];
$prependSpace = " ";