git.iko.soy/rowtype-yoga/purescript-native
on golang help sign in
Speeds up parsing by reading files as Text
c144a39Christoph Hegemann
hierarchy/Main.hs+2−2
@@ -33,7 +33,7 @@ import System.FilePath ((</>))
33import System.FilePath.Glob (glob)
33import System.FilePath.Glob (glob)
34import System.Exit (exitFailure, exitSuccess)
34import System.Exit (exitFailure, exitSuccess)
35import System.IO (hPutStr, stderr)
35import System.IO (hPutStr, stderr)
36−import System.IO.UTF8 (readUTF8File)
36+import System.IO.UTF8 (readUTF8FileT)
37
37
38import qualified Language.PureScript as P
38import qualified Language.PureScript as P
39import qualified Paths_purescript as Paths
39import qualified Paths_purescript as Paths
@@ -60,7 +60,7 @@ runModuleName (P.ModuleName pns) = intercalate "_" (P.runProperName `map` pns)
60
60
61readInput :: [FilePath] -> IO (Either P.MultipleErrors [P.Module])
61readInput :: [FilePath] -> IO (Either P.MultipleErrors [P.Module])
62readInput paths = do
62readInput paths = do
63− content <- mapM (\path -> (path, ) <$> readUTF8File path) paths
63+ content <- mapM (\path -> (path, ) <$> readUTF8FileT path) paths
64 return $ map snd <$> P.parseModulesFromFiles id content
64 return $ map snd <$> P.parseModulesFromFiles id content
65
65
66compile :: HierarchyOptions -> IO ()
66compile :: HierarchyOptions -> IO ()
psc-docs/Main.hs+4−3
@@ -7,6 +7,7 @@ import Control.Monad.Trans.Except (runExceptT)
7import Control.Arrow (first, second)
7import Control.Arrow (first, second)
8import Control.Category ((>>>))
8import Control.Category ((>>>))
9import Control.Monad.Writer
9import Control.Monad.Writer
10+import Data.Text (Text)
10import Data.Function (on)
11import Data.Function (on)
11import Data.List
12import Data.List
12import Data.Maybe (fromMaybe)
13import Data.Maybe (fromMaybe)
@@ -20,7 +21,7 @@ import qualified Language.PureScript as P
20import qualified Paths_purescript as Paths
21import qualified Paths_purescript as Paths
21import System.Exit (exitFailure)
22import System.Exit (exitFailure)
22import System.IO (hPutStrLn, hPrint, hSetEncoding, stderr, stdout, utf8)
23import System.IO (hPutStrLn, hPrint, hSetEncoding, stderr, stdout, utf8)
23−import System.IO.UTF8 (readUTF8File)
24+import System.IO.UTF8 (readUTF8FileT)
24import System.Directory (createDirectoryIfMissing)
25import System.Directory (createDirectoryIfMissing)
25import System.FilePath (takeDirectory)
26import System.FilePath (takeDirectory)
26import System.FilePath.Glob (glob)
27import System.FilePath.Glob (glob)
@@ -139,8 +140,8 @@ dumpTags input renderTags = do
139 ldump :: [String] -> IO ()
140 ldump :: [String] -> IO ()
140 ldump = mapM_ putStrLn
141 ldump = mapM_ putStrLn
141
142
142−parseFile :: FilePath -> IO (FilePath, String)
143+parseFile :: FilePath -> IO (FilePath, Text)
143−parseFile input = (,) input <$> readUTF8File input
144+parseFile input = (,) input <$> readUTF8FileT input
144
145
145inputFile :: Parser FilePath
146inputFile :: Parser FilePath
146inputFile = strArgument $
147inputFile = strArgument $
psc/Main.hs+4−3
@@ -15,6 +15,7 @@ import Data.Bool (bool)
15import qualified Data.ByteString.Lazy as B
15import qualified Data.ByteString.Lazy as B
16import qualified Data.ByteString.UTF8 as BU8
16import qualified Data.ByteString.UTF8 as BU8
17import qualified Data.Map as M
17import qualified Data.Map as M
18+import Data.Text (Text)
18import Data.Version (showVersion)
19import Data.Version (showVersion)
19
20
20import qualified Language.PureScript as P
21import qualified Language.PureScript as P
@@ -29,7 +30,7 @@ import qualified System.Console.ANSI as ANSI
29import System.Exit (exitSuccess, exitFailure)
30import System.Exit (exitSuccess, exitFailure)
30import System.FilePath.Glob (glob)
31import System.FilePath.Glob (glob)
31import System.IO (hSetEncoding, hPutStrLn, stdout, stderr, utf8)
32import System.IO (hSetEncoding, hPutStrLn, stdout, stderr, utf8)
32−import System.IO.UTF8
33+import System.IO.UTF8 (readUTF8FileT)
33
34
34data PSCMakeOptions = PSCMakeOptions
35data PSCMakeOptions = PSCMakeOptions
35 { pscmInput :: [FilePath]
36 { pscmInput :: [FilePath]
@@ -85,8 +86,8 @@ globWarningOnMisses warn = concatMapM globWithWarning
85 return paths
86 return paths
86 concatMapM f = fmap concat . mapM f
87 concatMapM f = fmap concat . mapM f
87
88
88−readInput :: [FilePath] -> IO [(FilePath, String)]
89+readInput :: [FilePath] -> IO [(FilePath, Text)]
89−readInput inputFiles = forM inputFiles $ \inFile -> (inFile, ) <$> readUTF8File inFile
90+readInput inputFiles = forM inputFiles $ \inFile -> (inFile, ) <$> readUTF8FileT inFile
90
91
91inputFile :: Parser FilePath
92inputFile :: Parser FilePath
92inputFile = strArgument $
93inputFile = strArgument $
purescript.cabal+4−1
@@ -334,6 +334,7 @@ executable psc
334 mtl -any,
334 mtl -any,
335 optparse-applicative >= 0.12.1,
335 optparse-applicative >= 0.12.1,
336 parsec -any,
336 parsec -any,
337+ text -any,
337 time -any,
338 time -any,
338 transformers -any,
339 transformers -any,
339 transformers-compat -any,
340 transformers-compat -any,
@@ -387,6 +388,7 @@ executable psc-docs
387 optparse-applicative >= 0.12.1,
388 optparse-applicative >= 0.12.1,
388 process -any,
389 process -any,
389 split -any,
390 split -any,
391+ text -any,
390 transformers -any,
392 transformers -any,
391 transformers-compat -any
393 transformers-compat -any
392 main-is: Main.hs
394 main-is: Main.hs
@@ -419,7 +421,8 @@ executable psc-hierarchy
419 mtl -any,
421 mtl -any,
420 optparse-applicative >= 0.12.1,
422 optparse-applicative >= 0.12.1,
421 parsec -any,
423 parsec -any,
422− process -any
424+ process -any,
425+ text -any
423 main-is: Main.hs
426 main-is: Main.hs
424 other-modules: Paths_purescript
427 other-modules: Paths_purescript
425 buildable: True
428 buildable: True
src/Language/PureScript/Docs/ParseAndBookmark.hs+6−7
@@ -9,13 +9,12 @@ import Control.Monad.Error.Class (MonadError(..))
9import Control.Monad.IO.Class (MonadIO(..))
9import Control.Monad.IO.Class (MonadIO(..))
10
10
11import qualified Data.Map as M
11import qualified Data.Map as M
12+import Data.Text (Text)
12
13
13import Language.PureScript.Docs.Convert (collectBookmarks)
14import Language.PureScript.Docs.Convert (collectBookmarks)
14import Language.PureScript.Docs.Types
15import Language.PureScript.Docs.Types
15import qualified Language.PureScript as P
16import qualified Language.PureScript as P
16−
17−import System.IO.UTF8 (readUTF8File)
17+import System.IO.UTF8 (readUTF8FileT)
18−
19import Web.Bower.PackageMeta (PackageName)
18import Web.Bower.PackageMeta (PackageName)
20
19
21-- |
20-- |
@@ -45,7 +44,7 @@ parseAndBookmark inputFiles depsFiles = do
45
44
46parseFiles ::
45parseFiles ::
47 (MonadError P.MultipleErrors m) =>
46 (MonadError P.MultipleErrors m) =>
48− [(FileInfo, FilePath)]
47+ [(FileInfo, Text)]
49 -> m [(FileInfo, P.Module)]
48 -> m [(FileInfo, P.Module)]
50parseFiles =
49parseFiles =
51 throwLeft . P.parseModulesFromFiles fileInfoToString
50 throwLeft . P.parseModulesFromFiles fileInfoToString
@@ -77,10 +76,10 @@ fileInfoToString :: FileInfo -> FilePath
77fileInfoToString (Local fn) = fn
76fileInfoToString (Local fn) = fn
78fileInfoToString (FromDep _ fn) = fn
77fileInfoToString (FromDep _ fn) = fn
79
78
80−parseFile :: FilePath -> IO (FilePath, String)
79+parseFile :: FilePath -> IO (FilePath, Text)
81−parseFile input' = (,) input' <$> readUTF8File input'
80+parseFile input' = (,) input' <$> readUTF8FileT input'
82
81
83−parseAs :: (MonadIO m) => (FilePath -> a) -> FilePath -> m (a, String)
82+parseAs :: (MonadIO m) => (FilePath -> a) -> FilePath -> m (a, Text)
84parseAs g = fmap (first g) . liftIO . parseFile
83parseAs g = fmap (first g) . liftIO . parseFile
85
84
86getDepsModuleNames :: [InPackage (FilePath, P.Module)] -> M.Map P.ModuleName PackageName
85getDepsModuleNames :: [InPackage (FilePath, P.Module)] -> M.Map P.ModuleName PackageName
src/Language/PureScript/Ide/Imports.hs+5−5
@@ -34,7 +34,6 @@ import Protolude
34import Control.Lens ((^.))
34import Control.Lens ((^.))
35import Data.List (findIndex, nubBy)
35import Data.List (findIndex, nubBy)
36import qualified Data.Text as T
36import qualified Data.Text as T
37−import qualified Data.Text.IO as TIO
38import qualified Language.PureScript as P
37import qualified Language.PureScript as P
39import Language.PureScript.Ide.Completion
38import Language.PureScript.Ide.Completion
40import Language.PureScript.Ide.Error
39import Language.PureScript.Ide.Error
@@ -42,6 +41,7 @@ import Language.PureScript.Ide.Filter
42import Language.PureScript.Ide.State
41import Language.PureScript.Ide.State
43import Language.PureScript.Ide.Types
42import Language.PureScript.Ide.Types
44import Language.PureScript.Ide.Util
43import Language.PureScript.Ide.Util
44+import System.IO.UTF8 (readUTF8FileT, writeUTF8FileT)
45
45
46data Import = Import P.ModuleName P.ImportDeclarationType (Maybe P.ModuleName)
46data Import = Import P.ModuleName P.ImportDeclarationType (Maybe P.ModuleName)
47 deriving (Eq, Show)
47 deriving (Eq, Show)
@@ -72,7 +72,7 @@ compImport (Import n i q) (Import n' i' q')
72parseImportsFromFile :: (MonadIO m, MonadError PscIdeError m) =>
72parseImportsFromFile :: (MonadIO m, MonadError PscIdeError m) =>
73 FilePath -> m (P.ModuleName, [Text], [Import], [Text])
73 FilePath -> m (P.ModuleName, [Text], [Import], [Text])
74parseImportsFromFile fp = do
74parseImportsFromFile fp = do
75− file <- liftIO (TIO.readFile fp)
75+ file <- liftIO (readUTF8FileT fp)
76 case sliceImportSection (T.lines file) of
76 case sliceImportSection (T.lines file) of
77 Right res -> pure res
77 Right res -> pure res
78 Left err -> throwError (GeneralError err)
78 Left err -> throwError (GeneralError err)
@@ -322,10 +322,10 @@ prettyPrintImportSection imports = map prettyPrintImport' (sort imports)
322answerRequest :: (MonadIO m) => Maybe FilePath -> [Text] -> m Success
322answerRequest :: (MonadIO m) => Maybe FilePath -> [Text] -> m Success
323answerRequest outfp rs =
323answerRequest outfp rs =
324 case outfp of
324 case outfp of
325− Nothing -> pure $ MultilineTextResult rs
325+ Nothing -> pure (MultilineTextResult rs)
326 Just outfp' -> do
326 Just outfp' -> do
327− liftIO $ TIO.writeFile outfp' (T.unlines rs)
327+ liftIO (writeUTF8FileT outfp' (T.unlines rs))
328− pure $ TextResult $ "Written to " <> T.pack outfp'
328+ pure (TextResult ("Written to " <> T.pack outfp'))
329
329
330-- | Test and ghci helper
330-- | Test and ghci helper
331parseImport :: Text -> Maybe Import
331parseImport :: Text -> Maybe Import
src/Language/PureScript/Ide/Rebuild.hs+2−2
@@ -21,7 +21,7 @@ import Language.PureScript.Ide.Error
21import Language.PureScript.Ide.State
21import Language.PureScript.Ide.State
22import Language.PureScript.Ide.Types
22import Language.PureScript.Ide.Types
23import Language.PureScript.Ide.Util
23import Language.PureScript.Ide.Util
24−import System.IO.UTF8 (readUTF8File)
24+import System.IO.UTF8 (readUTF8FileT)
25
25
26-- | Given a filepath performs the following steps:
26-- | Given a filepath performs the following steps:
27--
27--
@@ -44,7 +44,7 @@ rebuildFile
44 -> m Success
44 -> m Success
45rebuildFile path = do
45rebuildFile path = do
46
46
47− input <- liftIO (readUTF8File path)
47+ input <- liftIO (readUTF8FileT path)
48
48
49 m <- case snd <$> P.parseModuleFromFile identity (path, input) of
49 m <- case snd <$> P.parseModuleFromFile identity (path, input) of
50 Left parseError -> throwError
50 Left parseError -> throwError
src/Language/PureScript/Ide/SourceFile.hs+3−3
@@ -30,14 +30,14 @@ import qualified Language.PureScript as P
30import Language.PureScript.Ide.Error
30import Language.PureScript.Ide.Error
31import Language.PureScript.Ide.Types
31import Language.PureScript.Ide.Types
32import Language.PureScript.Ide.Util
32import Language.PureScript.Ide.Util
33−import System.IO.UTF8 (readUTF8File)
33+import System.IO.UTF8 (readUTF8FileT)
34
34
35parseModule
35parseModule
36 :: (MonadIO m)
36 :: (MonadIO m)
37 => FilePath
37 => FilePath
38− -> m (Either FilePath (FilePath, P.Module) )
38+ -> m (Either FilePath (FilePath, P.Module))
39parseModule path = do
39parseModule path = do
40− contents <- liftIO (readUTF8File path)
40+ contents <- liftIO (readUTF8FileT path)
41 case P.parseModuleFromFile identity (path, contents) of
41 case P.parseModuleFromFile identity (path, contents) of
42 Left _ -> pure (Left path)
42 Left _ -> pure (Left path)
43 Right m -> pure (Right m)
43 Right m -> pure (Right m)
src/Language/PureScript/Interactive/Module.hs+3−3
@@ -6,7 +6,7 @@ import Control.Monad
6import qualified Language.PureScript as P
6import qualified Language.PureScript as P
7import Language.PureScript.Interactive.Types
7import Language.PureScript.Interactive.Types
8import System.FilePath (pathSeparator)
8import System.FilePath (pathSeparator)
9−import System.IO.UTF8 (readUTF8File)
9+import System.IO.UTF8 (readUTF8FileT)
10
10
11-- * Support Module
11-- * Support Module
12
12
@@ -25,7 +25,7 @@ supportModuleIsDefined = any ((== supportModuleName) . P.getModuleName)
25--
25--
26loadModule :: FilePath -> IO (Either String [P.Module])
26loadModule :: FilePath -> IO (Either String [P.Module])
27loadModule filename = do
27loadModule filename = do
28− content <- readUTF8File filename
28+ content <- readUTF8FileT filename
29 return $ either (Left . P.prettyPrintMultipleErrors P.defaultPPEOptions) (Right . map snd) $ P.parseModulesFromFiles id [(filename, content)]
29 return $ either (Left . P.prettyPrintMultipleErrors P.defaultPPEOptions) (Right . map snd) $ P.parseModulesFromFiles id [(filename, content)]
30
30
31-- |
31-- |
@@ -34,7 +34,7 @@ loadModule filename = do
34loadAllModules :: [FilePath] -> IO (Either P.MultipleErrors [(FilePath, P.Module)])
34loadAllModules :: [FilePath] -> IO (Either P.MultipleErrors [(FilePath, P.Module)])
35loadAllModules files = do
35loadAllModules files = do
36 filesAndContent <- forM files $ \filename -> do
36 filesAndContent <- forM files $ \filename -> do
37− content <- readUTF8File filename
37+ content <- readUTF8FileT filename
38 return (filename, content)
38 return (filename, content)
39 return $ P.parseModulesFromFiles id filesAndContent
39 return $ P.parseModulesFromFiles id filesAndContent
40
40
src/Language/PureScript/Parser/Declarations.hs+4−3
@@ -19,6 +19,7 @@ import Prelude hiding (lex)
19
19
20import Data.Functor (($>))
20import Data.Functor (($>))
21import Data.Maybe (fromMaybe)
21import Data.Maybe (fromMaybe)
22+import Data.Text (Text)
22
23
23import Control.Applicative
24import Control.Applicative
24import Control.Arrow ((+++))
25import Control.Arrow ((+++))
@@ -281,7 +282,7 @@ parseModulesFromFiles
281 :: forall m k
282 :: forall m k
282 . MonadError MultipleErrors m
283 . MonadError MultipleErrors m
283 => (k -> FilePath)
284 => (k -> FilePath)
284− -> [(k, String)]
285+ -> [(k, Text)]
285 -> m [(k, Module)]
286 -> m [(k, Module)]
286parseModulesFromFiles toFilePath input =
287parseModulesFromFiles toFilePath input =
287 flip parU wrapError . inParallel . flip map input $ parseModuleFromFile toFilePath
288 flip parU wrapError . inParallel . flip map input $ parseModuleFromFile toFilePath
@@ -298,11 +299,11 @@ parseModulesFromFiles toFilePath input =
298-- | Parses a single module with FilePath for eventual parsing errors
299-- | Parses a single module with FilePath for eventual parsing errors
299parseModuleFromFile
300parseModuleFromFile
300 :: (k -> FilePath)
301 :: (k -> FilePath)
301− -> (k, String)
302+ -> (k, Text)
302 -> Either P.ParseError (k, Module)
303 -> Either P.ParseError (k, Module)
303parseModuleFromFile toFilePath (k, content) = do
304parseModuleFromFile toFilePath (k, content) = do
304 let filename = toFilePath k
305 let filename = toFilePath k
305− ts <- lex filename content
306+ ts <- lex' filename content
306 m <- runTokenParser filename parseModule ts
307 m <- runTokenParser filename parseModule ts
307 pure (k, m)
308 pure (k, m)
308
309
src/Language/PureScript/Parser/Lexer.hs+27−19
@@ -6,6 +6,7 @@ module Language.PureScript.Parser.Lexer
6 , Token()
6 , Token()
7 , TokenParser()
7 , TokenParser()
8 , lex
8 , lex
9+ , lex'
9 , anyToken
10 , anyToken
10 , token
11 , token
11 , match
12 , match
@@ -69,6 +70,8 @@ import Control.Monad (void, guard)
69
70
70import Data.Char (isSpace, isAscii, isSymbol, isAlphaNum)
71import Data.Char (isSpace, isAscii, isSymbol, isAlphaNum)
71import Data.Functor.Identity
72import Data.Functor.Identity
73+import Data.Text (Text)
74+import qualified Data.Text as T
72
75
73import Language.PureScript.Comments
76import Language.PureScript.Comments
74import Language.PureScript.Parser.State
77import Language.PureScript.Parser.State
@@ -154,8 +157,13 @@ data PositionedToken = PositionedToken
154instance Show PositionedToken where
157instance Show PositionedToken where
155 show = prettyPrintToken . ptToken
158 show = prettyPrintToken . ptToken
156
159
160+type Lexer u a = P.Parsec Text u a
161+
157lex :: FilePath -> String -> Either P.ParseError [PositionedToken]
162lex :: FilePath -> String -> Either P.ParseError [PositionedToken]
163+lex fp = lex' fp . T.pack
164+
165+lex' :: FilePath -> Text -> Either P.ParseError [PositionedToken]
158−lex f s = updatePositions <$> P.parse parseTokens f s
166+lex' f s = updatePositions <$> P.parse parseTokens f s
159
167
160updatePositions :: [PositionedToken] -> [PositionedToken]
168updatePositions :: [PositionedToken] -> [PositionedToken]
161updatePositions [] = []
169updatePositions [] = []
@@ -163,22 +171,22 @@ updatePositions (x:xs) = x : zipWith update (x:xs) xs
163 where
171 where
164 update PositionedToken { ptEndPos = pos } pt = pt { ptPrevEndPos = Just pos }
172 update PositionedToken { ptEndPos = pos } pt = pt { ptPrevEndPos = Just pos }
165
173
166−parseTokens :: P.Parsec String u [PositionedToken]
174+parseTokens :: Lexer u [PositionedToken]
167parseTokens = whitespace *> P.many parsePositionedToken <* P.skipMany parseComment <* P.eof
175parseTokens = whitespace *> P.many parsePositionedToken <* P.skipMany parseComment <* P.eof
168
176
169−whitespace :: P.Parsec String u ()
177+whitespace :: Lexer u ()
170whitespace = P.skipMany (P.satisfy isSpace)
178whitespace = P.skipMany (P.satisfy isSpace)
171
179
172−parseComment :: P.Parsec String u Comment
180+parseComment :: Lexer u Comment
173parseComment = (BlockComment <$> blockComment <|> LineComment <$> lineComment) <* whitespace
181parseComment = (BlockComment <$> blockComment <|> LineComment <$> lineComment) <* whitespace
174 where
182 where
175− blockComment :: P.Parsec String u String
183+ blockComment :: Lexer u String
176 blockComment = P.try $ P.string "{-" *> P.manyTill P.anyChar (P.try (P.string "-}"))
184 blockComment = P.try $ P.string "{-" *> P.manyTill P.anyChar (P.try (P.string "-}"))
177
185
178− lineComment :: P.Parsec String u String
186+ lineComment :: Lexer u String
179 lineComment = P.try $ P.string "--" *> P.manyTill P.anyChar (P.try (void (P.char '\n') <|> P.eof))
187 lineComment = P.try $ P.string "--" *> P.manyTill P.anyChar (P.try (void (P.char '\n') <|> P.eof))
180
188
181−parsePositionedToken :: P.Parsec String u PositionedToken
189+parsePositionedToken :: Lexer u PositionedToken
182parsePositionedToken = P.try $ do
190parsePositionedToken = P.try $ do
183 comments <- P.many parseComment
191 comments <- P.many parseComment
184 pos <- P.getPosition
192 pos <- P.getPosition
@@ -187,7 +195,7 @@ parsePositionedToken = P.try $ do
187 whitespace
195 whitespace
188 return $ PositionedToken pos pos' Nothing tok comments
196 return $ PositionedToken pos pos' Nothing tok comments
189
197
190−parseToken :: P.Parsec String u Token
198+parseToken :: Lexer u Token
191parseToken = P.choice
199parseToken = P.choice
192 [ P.try $ P.string "<-" *> P.notFollowedBy symbolChar *> pure LArrow
200 [ P.try $ P.string "<-" *> P.notFollowedBy symbolChar *> pure LArrow
193 , P.try $ P.string "←" *> P.notFollowedBy symbolChar *> pure LArrow
201 , P.try $ P.string "←" *> P.notFollowedBy symbolChar *> pure LArrow
@@ -226,34 +234,34 @@ parseToken = P.choice
226 ]
234 ]
227
235
228 where
236 where
229− parseLName :: P.Parsec String u String
237+ parseLName :: Lexer u String
230 parseLName = (:) <$> identStart <*> P.many identLetter
238 parseLName = (:) <$> identStart <*> P.many identLetter
231
239
232− parseUName :: P.Parsec String u String
240+ parseUName :: Lexer u String
233 parseUName = (:) <$> P.upper <*> P.many identLetter
241 parseUName = (:) <$> P.upper <*> P.many identLetter
234
242
235− parseSymbol :: P.Parsec String u String
243+ parseSymbol :: Lexer u String
236 parseSymbol = P.many1 symbolChar
244 parseSymbol = P.many1 symbolChar
237
245
238− identStart :: P.Parsec String u Char
246+ identStart :: Lexer u Char
239 identStart = P.lower <|> P.oneOf "_"
247 identStart = P.lower <|> P.oneOf "_"
240
248
241− identLetter :: P.Parsec String u Char
249+ identLetter :: Lexer u Char
242 identLetter = P.alphaNum <|> P.oneOf "_'"
250 identLetter = P.alphaNum <|> P.oneOf "_'"
243
251
244− symbolChar :: P.Parsec String u Char
252+ symbolChar :: Lexer u Char
245 symbolChar = P.satisfy isSymbolChar
253 symbolChar = P.satisfy isSymbolChar
246
254
247− parseCharLiteral :: P.Parsec String u Char
255+ parseCharLiteral :: Lexer u Char
248 parseCharLiteral = PT.charLiteral tokenParser
256 parseCharLiteral = PT.charLiteral tokenParser
249
257
250− parseStringLiteral :: P.Parsec String u String
258+ parseStringLiteral :: Lexer u String
251 parseStringLiteral = blockString <|> PT.stringLiteral tokenParser
259 parseStringLiteral = blockString <|> PT.stringLiteral tokenParser
252 where
260 where
253 delimiter = P.try (P.string "\"\"\"")
261 delimiter = P.try (P.string "\"\"\"")
254 blockString = delimiter >> P.manyTill P.anyChar delimiter
262 blockString = delimiter >> P.manyTill P.anyChar delimiter
255
263
256− parseNumber :: P.Parsec String u (Either Integer Double)
264+ parseNumber :: Lexer u (Either Integer Double)
257 parseNumber = (consumeLeadingZero >> P.parserZero) <|>
265 parseNumber = (consumeLeadingZero >> P.parserZero) <|>
258 (Right <$> P.try (PT.float tokenParser) <|>
266 (Right <$> P.try (PT.float tokenParser) <|>
259 Left <$> P.try (PT.natural tokenParser))
267 Left <$> P.try (PT.natural tokenParser))
@@ -267,7 +275,7 @@ parseToken = P.choice
267-- |
275-- |
268-- We use Text.Parsec.Token to implement the string and number lexemes
276-- We use Text.Parsec.Token to implement the string and number lexemes
269--
277--
270−langDef :: PT.GenLanguageDef String u Identity
278+langDef :: PT.GenLanguageDef Text u Identity
271langDef = PT.LanguageDef
279langDef = PT.LanguageDef
272 { PT.reservedNames = []
280 { PT.reservedNames = []
273 , PT.reservedOpNames = []
281 , PT.reservedOpNames = []
@@ -285,7 +293,7 @@ langDef = PT.LanguageDef
285-- |
293-- |
286-- A token parser based on the language definition
294-- A token parser based on the language definition
287--
295--
288−tokenParser :: PT.GenTokenParser String u Identity
296+tokenParser :: PT.GenTokenParser Text u Identity
289tokenParser = PT.makeTokenParser langDef
297tokenParser = PT.makeTokenParser langDef
290
298
291type TokenParser a = P.Parsec [PositionedToken] ParseState a
299type TokenParser a = P.Parsec [PositionedToken] ParseState a
src/System/IO/UTF8.hs+12−2
@@ -4,11 +4,21 @@ import Prelude.Compat
4
4
5import qualified Data.ByteString as BS
5import qualified Data.ByteString as BS
6import qualified Data.ByteString.UTF8 as UTF8
6import qualified Data.ByteString.UTF8 as UTF8
7+import Data.Text (Text)
8+import qualified Data.Text.Encoding as TE
9+
10+readUTF8FileT :: FilePath -> IO Text
11+readUTF8FileT inFile =
12+ fmap TE.decodeUtf8 (BS.readFile inFile)
13+
14+writeUTF8FileT :: FilePath -> Text -> IO ()
15+writeUTF8FileT inFile text =
16+ BS.writeFile inFile (TE.encodeUtf8 text)
7
17
8readUTF8File :: FilePath -> IO String
18readUTF8File :: FilePath -> IO String
9−readUTF8File inFile = do
19+readUTF8File inFile =
10 fmap UTF8.toString (BS.readFile inFile)
20 fmap UTF8.toString (BS.readFile inFile)
11
21
12writeUTF8File :: FilePath -> String -> IO ()
22writeUTF8File :: FilePath -> String -> IO ()
13−writeUTF8File inFile text = do
23+writeUTF8File inFile text =
14 BS.writeFile inFile (UTF8.fromString text)
24 BS.writeFile inFile (UTF8.fromString text)
tests/Language/PureScript/Ide/Imports/IntegrationSpec.hs+2−2
@@ -6,12 +6,12 @@ module Language.PureScript.Ide.Imports.IntegrationSpec where
6import Protolude
6import Protolude
7
7
8import qualified Data.Text as T
8import qualified Data.Text as T
9−import qualified Data.Text.IO as TIO
10import qualified Language.PureScript.Ide.Integration as Integration
9import qualified Language.PureScript.Ide.Integration as Integration
11import Test.Hspec
10import Test.Hspec
12
11
13import System.Directory
12import System.Directory
14import System.FilePath
13import System.FilePath
14+import System.IO.UTF8 (readUTF8FileT)
15
15
16setup :: IO ()
16setup :: IO ()
17setup = void (Integration.reset *> Integration.loadAll)
17setup = void (Integration.reset *> Integration.loadAll)
@@ -27,7 +27,7 @@ withSupportFiles test = do
27outputFileShouldBe :: [Text] -> IO ()
27outputFileShouldBe :: [Text] -> IO ()
28outputFileShouldBe expectation = do
28outputFileShouldBe expectation = do
29 outFp <- (</> "src" </> "ImportsSpecOut.tmp") <$> Integration.projectDirectory
29 outFp <- (</> "src" </> "ImportsSpecOut.tmp") <$> Integration.projectDirectory
30− outRes <- TIO.readFile outFp
30+ outRes <- readUTF8FileT outFp
31 shouldBe (T.lines outRes) expectation
31 shouldBe (T.lines outRes) expectation
32
32
33spec :: Spec
33spec :: Spec
tests/TestCompiler.hs+3−2
@@ -31,6 +31,7 @@ import Data.List (sort, stripPrefix, intercalate, groupBy, sortBy, minimumBy)
31import Data.Maybe (mapMaybe)
31import Data.Maybe (mapMaybe)
32import Data.Time.Clock (UTCTime())
32import Data.Time.Clock (UTCTime())
33import Data.Tuple (swap)
33import Data.Tuple (swap)
34+import qualified Data.Text as T
34
35
35import qualified Data.Map as M
36import qualified Data.Map as M
36
37
@@ -71,7 +72,7 @@ spec = do
71 supportPurs <- supportFiles "purs"
72 supportPurs <- supportFiles "purs"
72 supportPursFiles <- readInput supportPurs
73 supportPursFiles <- readInput supportPurs
73 supportExterns <- runExceptT $ do
74 supportExterns <- runExceptT $ do
74− modules <- ExceptT . return $ P.parseModulesFromFiles id supportPursFiles
75+ modules <- ExceptT . return $ P.parseModulesFromFiles id (map (fmap T.pack) supportPursFiles)
75 foreigns <- inferForeignModules modules
76 foreigns <- inferForeignModules modules
76 externs <- ExceptT . fmap fst . runTest $ P.make (makeActions foreigns) (map snd modules)
77 externs <- ExceptT . fmap fst . runTest $ P.make (makeActions foreigns) (map snd modules)
77 return (zip (map snd modules) externs)
78 return (zip (map snd modules) externs)
@@ -194,7 +195,7 @@ compile
194 -> IO (Either P.MultipleErrors [P.ExternsFile], P.MultipleErrors)
195 -> IO (Either P.MultipleErrors [P.ExternsFile], P.MultipleErrors)
195compile supportExterns inputFiles check = silence $ runTest $ do
196compile supportExterns inputFiles check = silence $ runTest $ do
196 fs <- liftIO $ readInput inputFiles
197 fs <- liftIO $ readInput inputFiles
197− ms <- P.parseModulesFromFiles id fs
198+ ms <- P.parseModulesFromFiles id (map (fmap T.pack) fs)
198 foreigns <- inferForeignModules ms
199 foreigns <- inferForeignModules ms
199 liftIO (check (map snd ms))
200 liftIO (check (map snd ms))
200 let actions = makeActions foreigns
201 let actions = makeActions foreigns