mirror of
https://github.com/moses-smt/mosesdecoder.git
synced 2024-08-16 15:00:33 +03:00
Merge pull request #190 from moses-smt/patch-detokenizer-ko
Korean sentences have spaces =)
This commit is contained in:
commit
5be48ce9db
@ -106,7 +106,7 @@ sub detokenize {
|
||||
my $prependSpace = " ";
|
||||
for ($i=0;$i<(scalar(@words));$i++) {
|
||||
if (&startsWithCJKChar($words[$i])) {
|
||||
if ($i > 0 && &endsWithCJKChar($words[$i-1])) {
|
||||
if (($i > 0 && &endsWithCJKChar($words[$i-1])) && ($language ne "ko")) {
|
||||
# perform left shift if this is a second consecutive CJK (Chinese/Japanese/Korean) word
|
||||
$text=$text.$words[$i];
|
||||
} else {
|
||||
|
Loading…
Reference in New Issue
Block a user