@@ -6,6 +6,7 @@ module Language.PureScript.Parser.Lexer
7 , TokenParser()
7 , TokenParser()
9 , anyToken
10 , anyToken
@@ -69,6 +70,8 @@ import Control.Monad (void, guard)
70import Data.Char (isSpace, isAscii, isSymbol, isAlphaNum)
71import Data.Char (isSpace, isAscii, isSymbol, isAlphaNum)
71import Data.Functor.Identity
72import Data.Functor.Identity
73+import Data.Text (Text)
74+import qualified Data.Text as T
73import Language.PureScript.Comments
76import Language.PureScript.Comments
74import Language.PureScript.Parser.State
77import Language.PureScript.Parser.State
@@ -154,8 +157,13 @@ data PositionedToken = PositionedToken
154instance Show PositionedToken where
157instance Show PositionedToken where
155 show = prettyPrintToken . ptToken
158 show = prettyPrintToken . ptToken
160+type Lexer u a = P.Parsec Text u a
157lex :: FilePath -> String -> Either P.ParseError [PositionedToken]
162lex :: FilePath -> String -> Either P.ParseError [PositionedToken]
163+lex fp = lex' fp . T.pack
165+lex' :: FilePath -> Text -> Either P.ParseError [PositionedToken]
158−lex f s = updatePositions <$> P.parse parseTokens f s
166+lex' f s = updatePositions <$> P.parse parseTokens f s
160updatePositions :: [PositionedToken] -> [PositionedToken]
168updatePositions :: [PositionedToken] -> [PositionedToken]
161updatePositions [] = []
169updatePositions [] = []
@@ -163,22 +171,22 @@ updatePositions (x:xs) = x : zipWith update (x:xs) xs
164 update PositionedToken { ptEndPos = pos } pt = pt { ptPrevEndPos = Just pos }
172 update PositionedToken { ptEndPos = pos } pt = pt { ptPrevEndPos = Just pos }
166−parseTokens :: P.Parsec String u [PositionedToken]
174+parseTokens :: Lexer u [PositionedToken]
167parseTokens = whitespace *> P.many parsePositionedToken <* P.skipMany parseComment <* P.eof
175parseTokens = whitespace *> P.many parsePositionedToken <* P.skipMany parseComment <* P.eof
169−whitespace :: P.Parsec String u ()
177+whitespace :: Lexer u ()
170whitespace = P.skipMany (P.satisfy isSpace)
178whitespace = P.skipMany (P.satisfy isSpace)
172−parseComment :: P.Parsec String u Comment
180+parseComment :: Lexer u Comment
173parseComment = (BlockComment <$> blockComment <|> LineComment <$> lineComment) <* whitespace
181parseComment = (BlockComment <$> blockComment <|> LineComment <$> lineComment) <* whitespace
175− blockComment :: P.Parsec String u String
183+ blockComment :: Lexer u String
176 blockComment = P.try $ P.string "{-" *> P.manyTill P.anyChar (P.try (P.string "-}"))
184 blockComment = P.try $ P.string "{-" *> P.manyTill P.anyChar (P.try (P.string "-}"))
178− lineComment :: P.Parsec String u String
186+ lineComment :: Lexer u String
179 lineComment = P.try $ P.string "--" *> P.manyTill P.anyChar (P.try (void (P.char '\n') <|> P.eof))
187 lineComment = P.try $ P.string "--" *> P.manyTill P.anyChar (P.try (void (P.char '\n') <|> P.eof))
181−parsePositionedToken :: P.Parsec String u PositionedToken
189+parsePositionedToken :: Lexer u PositionedToken
182parsePositionedToken = P.try $ do
190parsePositionedToken = P.try $ do
183 comments <- P.many parseComment
191 comments <- P.many parseComment
184 pos <- P.getPosition
192 pos <- P.getPosition
@@ -187,7 +195,7 @@ parsePositionedToken = P.try $ do
187 whitespace
195 whitespace
188 return $ PositionedToken pos pos' Nothing tok comments
196 return $ PositionedToken pos pos' Nothing tok comments
190−parseToken :: P.Parsec String u Token
198+parseToken :: Lexer u Token
191parseToken = P.choice
199parseToken = P.choice
192 [ P.try $ P.string "<-" *> P.notFollowedBy symbolChar *> pure LArrow
200 [ P.try $ P.string "<-" *> P.notFollowedBy symbolChar *> pure LArrow
193 , P.try $ P.string "←" *> P.notFollowedBy symbolChar *> pure LArrow
201 , P.try $ P.string "←" *> P.notFollowedBy symbolChar *> pure LArrow
@@ -226,34 +234,34 @@ parseToken = P.choice
229− parseLName :: P.Parsec String u String
237+ parseLName :: Lexer u String
230 parseLName = (:) <$> identStart <*> P.many identLetter
238 parseLName = (:) <$> identStart <*> P.many identLetter
232− parseUName :: P.Parsec String u String
240+ parseUName :: Lexer u String
233 parseUName = (:) <$> P.upper <*> P.many identLetter
241 parseUName = (:) <$> P.upper <*> P.many identLetter
235− parseSymbol :: P.Parsec String u String
243+ parseSymbol :: Lexer u String
236 parseSymbol = P.many1 symbolChar
244 parseSymbol = P.many1 symbolChar
238− identStart :: P.Parsec String u Char
246+ identStart :: Lexer u Char
239 identStart = P.lower <|> P.oneOf "_"
247 identStart = P.lower <|> P.oneOf "_"
241− identLetter :: P.Parsec String u Char
249+ identLetter :: Lexer u Char
242 identLetter = P.alphaNum <|> P.oneOf "_'"
250 identLetter = P.alphaNum <|> P.oneOf "_'"
244− symbolChar :: P.Parsec String u Char
252+ symbolChar :: Lexer u Char
245 symbolChar = P.satisfy isSymbolChar
253 symbolChar = P.satisfy isSymbolChar
247− parseCharLiteral :: P.Parsec String u Char
255+ parseCharLiteral :: Lexer u Char
248 parseCharLiteral = PT.charLiteral tokenParser
256 parseCharLiteral = PT.charLiteral tokenParser
250− parseStringLiteral :: P.Parsec String u String
258+ parseStringLiteral :: Lexer u String
251 parseStringLiteral = blockString <|> PT.stringLiteral tokenParser
259 parseStringLiteral = blockString <|> PT.stringLiteral tokenParser
253 delimiter = P.try (P.string "\"\"\"")
261 delimiter = P.try (P.string "\"\"\"")
254 blockString = delimiter >> P.manyTill P.anyChar delimiter
262 blockString = delimiter >> P.manyTill P.anyChar delimiter
256− parseNumber :: P.Parsec String u (Either Integer Double)
264+ parseNumber :: Lexer u (Either Integer Double)
257 parseNumber = (consumeLeadingZero >> P.parserZero) <|>
265 parseNumber = (consumeLeadingZero >> P.parserZero) <|>
258 (Right <$> P.try (PT.float tokenParser) <|>
266 (Right <$> P.try (PT.float tokenParser) <|>
259 Left <$> P.try (PT.natural tokenParser))
267 Left <$> P.try (PT.natural tokenParser))
@@ -267,7 +275,7 @@ parseToken = P.choice
268-- We use Text.Parsec.Token to implement the string and number lexemes
276-- We use Text.Parsec.Token to implement the string and number lexemes
270−langDef :: PT.GenLanguageDef String u Identity
278+langDef :: PT.GenLanguageDef Text u Identity
271langDef = PT.LanguageDef
279langDef = PT.LanguageDef
272 { PT.reservedNames = []
280 { PT.reservedNames = []
273 , PT.reservedOpNames = []
281 , PT.reservedOpNames = []
@@ -285,7 +293,7 @@ langDef = PT.LanguageDef
286-- A token parser based on the language definition
294-- A token parser based on the language definition
288−tokenParser :: PT.GenTokenParser String u Identity
296+tokenParser :: PT.GenTokenParser Text u Identity
289tokenParser = PT.makeTokenParser langDef
297tokenParser = PT.makeTokenParser langDef
291type TokenParser a = P.Parsec [PositionedToken] ParseState a
299type TokenParser a = P.Parsec [PositionedToken] ParseState a