moses-smt/mosesdecoder
1
1
Fork 0
mirror of https://github.com/moses-smt/mosesdecoder.git synced 2024-08-16 15:00:33 +03:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #190 from moses-smt/patch-detokenizer-ko

Browse Source 
Korean sentences have spaces =)
This commit is contained in:
Hieu Hoang 2018-01-19 10:54:50 +00:00 committed by GitHub
commit 5be48ce9db
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 1 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
scripts/tokenizer/detokenizer.perl
View File

@ -106,7 +106,7 @@ sub detokenize {
my $prependSpace = " ";
for ($i=0;$i<(scalar(@words));$i++) {
if (&startsWithCJKChar($words[$i])) {
if ($i > 0 && &endsWithCJKChar($words[$i-1])) {
if (($i > 0 && &endsWithCJKChar($words[$i-1])) && ($language ne "ko")) {
# perform left shift if this is a second consecutive CJK (Chinese/Japanese/Korean) word
$text=$text.$words[$i];
} else {