@@ -144,10 +144,15 @@ takeToken lex str
144 = do i <- scan lex 0 str -- i must be > 0 if successful
144 = do i <- scan lex 0 str -- i must be > 0 if successful
145 pure (substr 0 i (getString str), strTail i str)
145 pure (substr 0 i (getString str), strTail i str)
147−||| Recognise a digit 0-9
147+||| Recognise a single digit 0-9
152+||| Recognise one or more digits
149digits : Lexer
154digits : Lexer
150−digits = some (Pred isDigit)
155+digits = some digit
152||| Recognise a specific string
157||| Recognise a specific string
@@ -157,15 +162,75 @@ exact str with (unpack str)
157 exact str | (x :: xs)
162 exact str | (x :: xs)
158 = foldl SeqEmpty (is x) (map is xs)
163 = foldl SeqEmpty (is x) (map is xs)
165+||| Recognise a single hexidecimal digit
168+hexDigit = digit <|> oneOf "abcdefABCDEF"
170+||| Recognise one or more hexidecimal digits
173+hexDigits = some hexDigit
175+||| Recognise a single alpha character
180+||| Recognise one or more alpha characters
185+||| Recognise a lowercase alpha character
190+||| Recognise one or more lowercase alpha characters
195+||| Recognise an uppercase alpha character
200+||| Recognise one or more uppercase alpha characters
205+||| Recognise an alphanumeric character
208+alphaNum = pred isAlphaNum
210+||| Recognise one or more alphanumeric characters
213+alphaNums = some alphaNum
160−||| Recognise a whitespace character
215+||| Recognise a single whitespace character
162space : Lexer
217space : Lexer
163−space = some (pred isSpace)
218+space = pred isSpace
220+||| Recognise one or more whitespace characters
165−||| Recognise a non-alphanumeric, non-whitespace character
225+||| Recognise a single non-whitespace, non-alphanumeric character
167symbol : Lexer
227symbol : Lexer
168−symbol = some (pred (\x => not (isAlphaNum x) && not (isSpace x)))
228+symbol = pred (\x => not (isSpace x || isAlphaNum x))
230+||| Recognise one or more non-whitespace, non-alphanumeric characters
233+symbols = some symbol
170||| Recognise zero or more occurrences of a sub-lexer between
235||| Recognise zero or more occurrences of a sub-lexer between
171||| delimiting lexers
236||| delimiting lexers
@@ -202,6 +267,11 @@ export
202intLit : Lexer
267intLit : Lexer
203intLit = opt (is '-') <+> digits
268intLit = opt (is '-') <+> digits
270+||| Recognise a hexidecimal literal, prefixed by "0x" or "0X"
273+hexLit = is '0' <+> oneOf "xX" <+> hexDigits
205||| A mapping from lexers to the tokens they produce.
275||| A mapping from lexers to the tokens they produce.
206||| This is a list of pairs `(Lexer, String -> tokenType)`
276||| This is a list of pairs `(Lexer, String -> tokenType)`
207||| For each Lexer in the list, if a substring in the input matches, run
277||| For each Lexer in the list, if a substring in the input matches, run