From c6c3bc84b7673618f379482cbc6b708f55a9ecd3 Mon Sep 17 00:00:00 2001 From: alvations Date: Fri, 23 Dec 2016 14:21:20 +0800 Subject: [PATCH] Changed \p{Hyphen} to \p{LineBreak} Using Perl v5.18.2, it's reporting this warning: **Use of 'Hyphen' in \p{} or \P{} is deprecated because: Supplanted by Line_Break property values; see www.unicode.org/reports/tr14** --- scripts/generic/mteval-v13a.pl | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scripts/generic/mteval-v13a.pl b/scripts/generic/mteval-v13a.pl index bdc2d9479..2e5d29ad5 100755 --- a/scripts/generic/mteval-v13a.pl +++ b/scripts/generic/mteval-v13a.pl @@ -19,6 +19,8 @@ binmode STDERR, ":utf8"; # version 13a # * modified the scoring functions to prevent division-by-zero errors when a system segment is empty # * affected methods: 'bleu_score' and 'bleu_score_smoothing' +# * use \p{Line_Breaks} instead of \p{Hyphen} when stripping end-of-line hyphenation and join lines +# * because \p{Hyphen} is deprecated since 2016-06-01, see http://www.unicode.org/reports/tr14/#Hyphen # # version 13 # * Uses a XML parser to read data (only when extension is .xml) @@ -948,7 +950,7 @@ sub tokenization_international my ($norm_text) = @_; $norm_text =~ s///g; # strip "skipped" tags - $norm_text =~ s/\p{Hyphen}\p{Zl}//g; # strip end-of-line hyphenation and join lines + $norm_text =~ s/\p{Line_Break}\p{Zl}//g; # strip end-of-line hyphenation and join lines $norm_text =~ s/\p{Zl}/ /g; # join lines # replace entities