fixed text encoding issues

node/src/Container.hs

@@ -10,7 +10,7 @@ import Data.Bytes.Serial (serialize)
 import Data.Text.Encoding (decodeUtf8)
 import Network.HTTP.Types.Method (StdMethod(OPTIONS))
 import Network.Wai.Middleware.RequestLogger (logStdoutDev)
-import System.IO (hSetBinaryMode, hFlush, stdin)
+import System.IO (hSetBinaryMode, hFlush, hSetEncoding, stdin, stdout, stderr, utf8)
 import System.Process as P
 import Unison.NodeProtocol.V0 (protocol)
 import Unison.NodeServer as NS
@@ -64,6 +64,7 @@ main = Mux.uniqueChannel >>= \rand ->
         P.std_in = P.CreatePipe,
         P.std_err = P.CreatePipe }
   in do
+    mapM_ (`hSetEncoding` utf8) [stdout, stdin, stderr]
 #ifdef leveldb
     putStrLn "using leveldb-based block store"
 #else

node/src/Node.hs

@@ -3,6 +3,7 @@
 
 module Main where
 
+import System.IO
 import Unison.Hash.Extra ()
 import Unison.Node.Store (Store)
 import Unison.Reference (Reference)
@@ -44,6 +45,7 @@ makeRandomAddress crypt = Address <$> C.randomBytes crypt 64
 
 main :: IO ()
 main = do
+  mapM_ (`hSetEncoding` utf8) [stdout, stdin, stderr]
   store' <- store
   logger <- L.atomic (L.atInfo L.toStandardError)
   let crypto = C.noop "dummypublickey"

node/src/Worker.hs

@@ -2,14 +2,16 @@
 
 module Main where
 
+import Data.Text.Encoding (decodeUtf8)
 import Control.Concurrent.STM.TVar
 import Control.Monad
-import System.IO (stderr)
+import System.IO (stderr, stdin, stdout, hSetEncoding, utf8)
 import Unison.Hash (Hash)
 import Unison.NodeProtocol.V0 (protocol)
 import Unison.NodeWorker as W
 import Unison.SerializationAndHashing (TermV)
 import qualified Control.Concurrent.STM as STM
+import qualified Data.ByteString as B
 import qualified Data.Set as Set
 import qualified Data.Text as Text
 import qualified Data.Text.IO as Text.IO
@@ -31,6 +33,7 @@ import qualified Unison.Util.Logger as L
 
 main :: IO ()
 main = do
+  mapM_ (`hSetEncoding` utf8) [stdout, stdin, stderr]
   logger <- L.scope "worker-main" <$> Config.loggerTo stderr
   W.make protocol crypto (pure $ lang logger) where
   crypto keypair = C.noop (W.public keypair)
@@ -90,7 +93,7 @@ main = do
       unRemote _ = Nothing
       remote = Term.remote
       loadDeclarations path node = do
-        txt <- Text.IO.readFile path
+        txt <- decodeUtf8 <$> B.readFile path
         let str = Text.unpack txt
         r <- Note.run $ Node.declare' Term.ref str node
         L.info logger $ "loaded " ++ path

node/tests/Unison/Test/NodeUtil.hs

@@ -3,6 +3,7 @@
 module Unison.Test.NodeUtil where
 
 import Control.Applicative
+import Data.Text.Encoding (decodeUtf8)
 import Unison.Hash (Hash)
 import Unison.Node (Node)
 import Unison.Reference (Reference)
@@ -11,6 +12,7 @@ import Unison.Symbol (Symbol)
 import Unison.Term (Term)
 import Unison.Type (Type)
 import Unison.Var (Var)
+import qualified Data.ByteString as B
 import qualified Data.Text as Text
 import qualified Data.Text.IO as Text.IO
 import qualified System.FilePath as FP
@@ -48,7 +50,7 @@ loadDeclarations :: L.Logger -> FilePath -> Node IO V Reference (Type V) (Term V
 loadDeclarations logger path node = do
   -- note - when run from repl current directory is root, but when run via stack test, current
   -- directory is the shared subdir - so we check both locations
-  txt <- Text.IO.readFile path <|> Text.IO.readFile (".." `FP.combine` path)
+  txt <- decodeUtf8 <$> (B.readFile path <|> B.readFile (".." `FP.combine` path))
   let str = Text.unpack txt
   _ <- Note.run $ Node.declare' Term.ref str node
   L.info logger $ "loaded file: " ++ path

shared/tests/Suite.hs

@@ -1,5 +1,6 @@
 module Main where
 
+import System.IO
 import Test.Tasty
 import qualified Unison.Test.Doc as Doc
 import qualified Unison.Test.Typechecker as Typechecker
@@ -13,4 +14,6 @@ tests :: TestTree
 tests = testGroup "unison" [Doc.tests, Typechecker.tests, Term.tests, TermParser.tests, TypeParser.tests, Interpreter.tests, Components.tests]
 
 main :: IO ()
-main = defaultMain tests
+main = do
+  mapM_ (`hSetEncoding` utf8) [stdout, stdin, stderr]
+  defaultMain tests

shared/tests/Unison/Test/Common.hs

@@ -4,6 +4,7 @@ module Unison.Test.Common where
 import Control.Applicative
 import Control.Monad.IO.Class
 import Data.Foldable
+import Data.Text.Encoding (decodeUtf8)
 import System.IO (FilePath)
 import Unison.Symbol (Symbol)
 import Unison.Node (Node)
@@ -11,6 +12,7 @@ import Unison.Reference (Reference)
 import Unison.Term (Term)
 import Unison.Type (Type)
 import Unison.Views (defaultSymbol)
+import qualified Data.ByteString as B
 import qualified Data.Map as Map
 import qualified Data.Text.IO as Text.IO
 import qualified Data.Text as Text
@@ -31,7 +33,7 @@ loadDeclarations :: FilePath -> Node IO V Reference (Type V) (Term V) -> IO ()
 loadDeclarations path node = do
   -- note - when run from repl current directory is root, but when run via stack test, current
   -- directory is the shared subdir - so we check both locations
-  txt <- Text.IO.readFile path <|> Text.IO.readFile (".." `FP.combine` path)
+  txt <- decodeUtf8 <$> (B.readFile path <|> B.readFile (".." `FP.combine` path))
   let str = Text.unpack txt
   _ <- Note.run $ Node.declare' Term.ref str node
   putStrLn $ "loaded file: " ++ path

shared/unison-shared.cabal

@@ -111,6 +111,7 @@ test-suite tests
   other-modules:
   build-depends:
     base,
+    bytestring,
     containers,
     filepath,
     tasty,