From 90e46ef3403b399e31382dd5586551d675b73109 Mon Sep 17 00:00:00 2001 From: Steve Dunham Date: Mon, 5 Sep 2022 11:10:45 -0700 Subject: [PATCH] Add test case for lexer issue and fix other instance. --- src/Libraries/Text/Lexer/Core.idr | 2 +- tests/contrib/lexer/Test.idr | 23 +++++++++++++++++++++++ tests/contrib/lexer/expected | 11 +++++++++++ tests/contrib/lexer/run | 3 +++ 4 files changed, 38 insertions(+), 1 deletion(-) create mode 100644 tests/contrib/lexer/Test.idr create mode 100644 tests/contrib/lexer/expected create mode 100644 tests/contrib/lexer/run diff --git a/src/Libraries/Text/Lexer/Core.idr b/src/Libraries/Text/Lexer/Core.idr index c0f7425b9..dacefdebf 100644 --- a/src/Libraries/Text/Lexer/Core.idr +++ b/src/Libraries/Text/Lexer/Core.idr @@ -155,7 +155,7 @@ tokenise pred line col acc tmap str getCols : List Char -> Int -> Int getCols x c - = case span (/= '\n') (reverse x) of + = case span (/= '\n') x of (incol, []) => c + cast (length incol) (incol, _) => cast (length incol) diff --git a/tests/contrib/lexer/Test.idr b/tests/contrib/lexer/Test.idr new file mode 100644 index 000000000..a2bbdf00c --- /dev/null +++ b/tests/contrib/lexer/Test.idr @@ -0,0 +1,23 @@ +import Text.Lexer + +data Kind = Ignore | Ident | Oper | Number + +ignore : WithBounds (Token Kind) -> Bool +ignore (MkBounded (Tok Ignore _) _ _) = True +ignore _ = False + +tmap : TokenMap (Token Kind) +tmap = [ + (alpha <+> many alphaNum, Tok Ident), + (some digit, Tok Number), + (some symbol, Tok Oper), + (spaces, Tok Ignore) +] + +show : WithBounds (Token Kind) -> String +show t@(MkBounded (Tok _ v) _ _) = "\{show $ start t} \{show v}" + +main : IO () +main = do + let toks = filter (not . ignore) $ fst $ lex tmap "let x = 1\n y = 2\n in x + y" + traverse_ (putStrLn . show) toks diff --git a/tests/contrib/lexer/expected b/tests/contrib/lexer/expected new file mode 100644 index 000000000..cea48d8cc --- /dev/null +++ b/tests/contrib/lexer/expected @@ -0,0 +1,11 @@ +(0, 0) "let" +(0, 4) "x" +(0, 6) "=" +(0, 8) "1" +(1, 4) "y" +(1, 6) "=" +(1, 8) "2" +(2, 1) "in" +(2, 4) "x" +(2, 6) "+" +(2, 8) "y" diff --git a/tests/contrib/lexer/run b/tests/contrib/lexer/run new file mode 100644 index 000000000..e45756d6b --- /dev/null +++ b/tests/contrib/lexer/run @@ -0,0 +1,3 @@ +$1 --no-banner --no-color --console-width 0 -p contrib -x main Test.idr + +rm -rf build