moses-smt/mosesdecoder
1
1
Fork 0
mirror of https://github.com/moses-smt/mosesdecoder.git synced 2024-12-24 20:32:59 +03:00
Code Issues Packages Projects Releases Wiki Activity

Korean words has spaces =)

Browse Source
This commit is contained in:
alvations 2018-01-19 13:29:53 +08:00 committed by GitHub
parent 3a0631a05b
commit 194964c017
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 1 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
scripts/tokenizer/detokenizer.perl
View File

@ -106,7 +106,7 @@ sub detokenize {
my $prependSpace = " ";
for ($i=0;$i<(scalar(@words));$i++) {
if (&startsWithCJKChar($words[$i])) {
if ($i > 0 && &endsWithCJKChar($words[$i-1])) {
if (($i > 0 && &endsWithCJKChar($words[$i-1])) && ($language ne "ko")) {
# perform left shift if this is a second consecutive CJK (Chinese/Japanese/Korean) word
$text=$text.$words[$i];
} else {