Add newline conversion to utf8 file helpers (#2943)

lib/unison-prelude/src/Unison/Prelude.hs

@@ -57,6 +57,8 @@ import Safe as X (atMay, headMay, lastMay, readMay)
 import Text.Read as X (readMaybe)
 import Control.Monad.Trans.Except (ExceptT (ExceptT), runExceptT, withExceptT)
 import qualified UnliftIO
+import qualified System.IO as IO
+import qualified GHC.IO.Handle as Handle
 
 onNothing :: Applicative m => m a -> Maybe a -> m a
 onNothing x =
@@ -85,22 +87,43 @@ throwEitherMWith f action = throwExceptT . withExceptT f $ (ExceptT action)
 tShow :: Show a => a -> Text
 tShow = Text.pack . show
 
--- Read an entire file strictly assuming UTF8
+-- | Strictly read an entire file decoding UTF8.
+-- Converts \r\n -> \n on windows.
 readUtf8 :: FilePath -> IO Text
-readUtf8 p = decodeUtf8 <$> BS.readFile p
+readUtf8 fileName =
+  UnliftIO.withFile fileName UnliftIO.ReadMode readUtf8Handle
 
+-- | Strictly read from a handle, decoding UTF8, or failing if not valid UTF8
+-- Converts \r\n -> \n on windows.
 safeReadUtf8 :: FilePath -> IO (Either IOException Text)
 safeReadUtf8 p = try (readUtf8 p)
 
+-- | Strictly read from a handle, decoding UTF8.
+-- Note, this changes the newline-mode of the handle
+-- to convert \r\n -> \n on windows.
+readUtf8Handle :: IO.Handle -> IO Text
+readUtf8Handle handle = do
+  IO.hSetNewlineMode handle IO.universalNewlineMode
+  decodeUtf8 <$> BS.hGetContents handle
+
+-- | Strictly read from stdin, decoding UTF8.
+-- Converts \r\n -> \n on windows.
 safeReadUtf8StdIn :: IO (Either IOException Text)
-safeReadUtf8StdIn = try $ decodeUtf8 <$> BS.getContents
+safeReadUtf8StdIn = do
+  handle <- Handle.hDuplicate IO.stdin
+  try $ readUtf8Handle handle
 
 uncurry4 :: (a -> b -> c -> d -> e) -> (a, b, c, d) -> e
 uncurry4 f (a, b, c, d) =
   f a b c d
 
+-- | Write a file strictly assuming UTF8
+-- Converts \n -> \r\n on windows.
 writeUtf8 :: FilePath -> Text -> IO ()
-writeUtf8 p txt = BS.writeFile p (encodeUtf8 txt)
+writeUtf8 fileName txt = do
+  UnliftIO.withFile fileName UnliftIO.WriteMode $ \handle -> do
+    IO.hSetNewlineMode handle IO.universalNewlineMode
+    BS.hPut handle (encodeUtf8 txt)
 
 reportBug :: String -> String -> String
 reportBug bugId msg =

parser-typechecker/src/Unison/Parsers.hs

@@ -4,8 +4,6 @@ module Unison.Parsers where
 import Unison.Prelude
 
 import qualified Data.Text                     as Text
-import           Data.Text.IO                   ( readFile )
-import           Prelude                 hiding ( readFile )
 import qualified Unison.NamesWithHistory                 as Names
 import qualified Unison.Builtin                as Builtin
 import qualified Unison.FileParser             as FileParser
@@ -62,7 +60,7 @@ readAndParseFile
   -> FilePath
   -> IO (Either (Parser.Err v) (UnisonFile v Ann))
 readAndParseFile penv fileName = do
-  txt <- readFile fileName
+  txt <- readUtf8 fileName
   let src = Text.unpack txt
   pure $ parseFile fileName src penv
 
@@ -72,7 +70,7 @@ unsafeParseTerm s = fmap (unsafeGetRightFrom s) . parseTerm $ s
 unsafeReadAndParseFile
   :: Parser.ParsingEnv -> FilePath -> IO (UnisonFile Symbol Ann)
 unsafeReadAndParseFile penv fileName = do
-  txt <- readFile fileName
+  txt <- readUtf8 fileName
   let str = Text.unpack txt
   pure . unsafeGetRightFrom str $ parseFile fileName str penv
 

unison-cli/src/Unison/CommandLine/Main.hs

@@ -14,7 +14,6 @@ import Control.Exception (finally, catch)
 import Control.Monad.State (runStateT)
 import Data.Configurator.Types (Config)
 import Data.IORef
-import Prelude hiding (readFile, writeFile)
 import System.IO.Error (isDoesNotExistError)
 import Unison.Codebase.Branch (Branch)
 import qualified Unison.Codebase.Branch as Branch
@@ -33,7 +32,6 @@ import Unison.Symbol (Symbol)
 import qualified Control.Concurrent.Async as Async
 import qualified Data.Map as Map
 import qualified Data.Text as Text
-import qualified Data.Text.IO
 import qualified System.Console.Haskeline as Line
 import qualified Crypto.Random        as Random
 import qualified Unison.Codebase.Path as Path
@@ -148,7 +146,7 @@ main dir welcome initialPath (config, cancelConfig) initialInputs runtime codeba
                       _ | isDoesNotExistError e -> return InvalidSourceNameError
                       _ -> return LoadError
                   go = do
-                    contents <- Data.Text.IO.readFile $ Text.unpack fname
+                    contents <- readUtf8 $ Text.unpack fname
                     return $ LoadSuccess contents
                   in catch go handle
             else return InvalidSourceNameError

unison-cli/src/Unison/CommandLine/OutputMessages.hs

@@ -19,7 +19,6 @@ import qualified Data.Sequence as Seq
 import qualified Data.Set as Set
 import Data.Set.NonEmpty (NESet)
 import qualified Data.Text as Text
-import Data.Text.IO (readFile, writeFile)
 import Data.Tuple (swap)
 import Data.Tuple.Extra (dupe, uncurry3)
 import System.Directory
@@ -125,7 +124,6 @@ import qualified Unison.Util.Relation as R
 import Unison.Var (Var)
 import qualified Unison.Var as Var
 import qualified Unison.WatchKind as WK
-import Prelude hiding (readFile, writeFile)
 import qualified Data.List.NonEmpty as NEList
 import qualified U.Util.Monoid as Monoid
 import qualified Data.Foldable as Foldable
@@ -1651,9 +1649,9 @@ displayRendered outputLoc pp =
           existingContents <- do
             exists <- doesFileExist path
             if exists
-              then readFile path
+              then readUtf8 path
               else pure ""
-          writeFile path . Text.pack . P.toPlain 80 $
+          writeUtf8 path . Text.pack . P.toPlain 80 $
             P.lines [pp, "", P.text existingContents]
         message pp path =
           P.callout "☝️" $
@@ -1687,9 +1685,9 @@ displayDefinitions outputLoc ppe types terms =
           existingContents <- do
             exists <- doesFileExist path
             if exists
-              then readFile path
+              then readUtf8 path
               else pure ""
-          writeFile path . Text.pack . P.toPlain 80 $
+          writeUtf8 path . Text.pack . P.toPlain 80 $
             P.lines
               [ code,
                 "",