From fdb7384d3d401cc9d6ece34a038e1b585712c512 Mon Sep 17 00:00:00 2001
From: Joel Barry <joelb@isi.edu>
Date: Wed, 27 Feb 2019 10:17:29 -0500
Subject: [PATCH] Fix non-ASCII lowercasing

---
 scripts/generic/multi-bleu-detok.perl | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/scripts/generic/multi-bleu-detok.perl b/scripts/generic/multi-bleu-detok.perl
index d2ef60c90..e1f69501e 100755
--- a/scripts/generic/multi-bleu-detok.perl
+++ b/scripts/generic/multi-bleu-detok.perl
@@ -14,6 +14,9 @@
 use warnings;
 use strict;
 
+binmode(STDIN, ":utf8");
+use open ':encoding(UTF-8)';
+
 my $lowercase = 0;
 if ($ARGV[0] eq "-lc") {
   $lowercase = 1;