2020-09-12 14:42:10 +03:00
|
|
|
|
#!/bin/bash
|
|
|
|
|
#
|
2020-09-12 16:16:01 +03:00
|
|
|
|
# USAGE preprocess.sh source-langid target-langid spmodel [noflags] < input > output
|
2020-09-12 14:42:10 +03:00
|
|
|
|
#
|
|
|
|
|
# replace SPMENCODE with your own setup!
|
2021-02-25 18:17:21 +03:00
|
|
|
|
#
|
|
|
|
|
# CHANGES
|
|
|
|
|
#
|
|
|
|
|
# * issue with perl code that removes control characters
|
|
|
|
|
# unicode property Other = \p{C}) seems to remove
|
|
|
|
|
# newline characters as well --> add negative lookahead
|
|
|
|
|
# to avoid removing newline characters!
|
|
|
|
|
|
2020-09-12 14:42:10 +03:00
|
|
|
|
|
2021-03-30 00:00:28 +03:00
|
|
|
|
SPMENCODE=`which spm_encode || echo "${PWD}/tools/marian-dev/build/spm_encode"`
|
2020-09-12 14:42:10 +03:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
if [ "$4" == "noflags" ]; then
|
|
|
|
|
sed -e 's/,/,/g' \
|
|
|
|
|
-e 's/。 */. /g' \
|
|
|
|
|
-e 's/、/,/g' \
|
|
|
|
|
-e 's/”/"/g' \
|
|
|
|
|
-e 's/“/"/g' \
|
|
|
|
|
-e 's/∶/:/g' \
|
|
|
|
|
-e 's/:/:/g' \
|
|
|
|
|
-e 's/?/\?/g' \
|
|
|
|
|
-e 's/《/"/g' \
|
|
|
|
|
-e 's/》/"/g' \
|
|
|
|
|
-e 's/)/\)/g' \
|
|
|
|
|
-e 's/!/\!/g' \
|
|
|
|
|
-e 's/(/\(/g' \
|
|
|
|
|
-e 's/;/;/g' \
|
|
|
|
|
-e 's/1/"/g' \
|
|
|
|
|
-e 's/」/"/g' \
|
|
|
|
|
-e 's/「/"/g' \
|
|
|
|
|
-e 's/0/0/g' \
|
|
|
|
|
-e 's/3/3/g' \
|
|
|
|
|
-e 's/2/2/g' \
|
|
|
|
|
-e 's/5/5/g' \
|
|
|
|
|
-e 's/6/6/g' \
|
|
|
|
|
-e 's/9/9/g' \
|
|
|
|
|
-e 's/7/7/g' \
|
|
|
|
|
-e 's/8/8/g' \
|
|
|
|
|
-e 's/4/4/g' \
|
|
|
|
|
-e 's/. */. /g' \
|
|
|
|
|
-e 's/~/\~/g' \
|
|
|
|
|
-e "s/’/\'/g" \
|
|
|
|
|
-e 's/…/\.\.\./g' \
|
|
|
|
|
-e 's/━/\-/g' \
|
|
|
|
|
-e 's/〈/\</g' \
|
|
|
|
|
-e 's/〉/\>/g' \
|
|
|
|
|
-e 's/【/\[/g' \
|
|
|
|
|
-e 's/】/\]/g' \
|
|
|
|
|
-e 's/%/\%/g' |
|
|
|
|
|
perl -C -pe 's/\p{C}/ /g;' |
|
|
|
|
|
sed 's/ */ /g;s/^ *//g;s/ *$//g' |
|
|
|
|
|
${SPMENCODE} --model $3
|
|
|
|
|
else
|
|
|
|
|
sed -e 's/,/,/g' \
|
|
|
|
|
-e 's/。 */. /g' \
|
|
|
|
|
-e 's/、/,/g' \
|
|
|
|
|
-e 's/”/"/g' \
|
|
|
|
|
-e 's/“/"/g' \
|
|
|
|
|
-e 's/∶/:/g' \
|
|
|
|
|
-e 's/:/:/g' \
|
|
|
|
|
-e 's/?/\?/g' \
|
|
|
|
|
-e 's/《/"/g' \
|
|
|
|
|
-e 's/》/"/g' \
|
|
|
|
|
-e 's/)/\)/g' \
|
|
|
|
|
-e 's/!/\!/g' \
|
|
|
|
|
-e 's/(/\(/g' \
|
|
|
|
|
-e 's/;/;/g' \
|
|
|
|
|
-e 's/1/"/g' \
|
|
|
|
|
-e 's/」/"/g' \
|
|
|
|
|
-e 's/「/"/g' \
|
|
|
|
|
-e 's/0/0/g' \
|
|
|
|
|
-e 's/3/3/g' \
|
|
|
|
|
-e 's/2/2/g' \
|
|
|
|
|
-e 's/5/5/g' \
|
|
|
|
|
-e 's/6/6/g' \
|
|
|
|
|
-e 's/9/9/g' \
|
|
|
|
|
-e 's/7/7/g' \
|
|
|
|
|
-e 's/8/8/g' \
|
|
|
|
|
-e 's/4/4/g' \
|
|
|
|
|
-e 's/. */. /g' \
|
|
|
|
|
-e 's/~/\~/g' \
|
|
|
|
|
-e "s/’/\'/g" \
|
|
|
|
|
-e 's/…/\.\.\./g' \
|
|
|
|
|
-e 's/━/\-/g' \
|
|
|
|
|
-e 's/〈/\</g' \
|
|
|
|
|
-e 's/〉/\>/g' \
|
|
|
|
|
-e 's/【/\[/g' \
|
|
|
|
|
-e 's/】/\]/g' \
|
|
|
|
|
-e 's/%/\%/g' |
|
2021-03-30 00:00:28 +03:00
|
|
|
|
perl -C -pe 's/(?!\n)\p{C}/ /g;' |
|
2020-09-12 14:42:10 +03:00
|
|
|
|
sed 's/ */ /g;s/^ *//g;s/ *$//g' |
|
|
|
|
|
${SPMENCODE} --model $3 |
|
|
|
|
|
sed "s/^/>>$2<< /"
|
|
|
|
|
fi
|
|
|
|
|
|