@@ -144,10 +144,15 @@ takeToken lex str
144144 = do i <- scan lex 0 str -- i must be > 0 if successful
145145 pure (substr 0 i (getString str), strTail i str)
146146
147−||| Recognise a digit 0-9
147+||| Recognise a single digit 0-9
148+export
149+digit : Lexer
150+digit = pred isDigit
151+
152+||| Recognise one or more digits
148153export
149154digits : Lexer
150−digits = some (Pred isDigit)
155+digits = some digit
151156
152157||| Recognise a specific string
153158export
@@ -157,15 +162,75 @@ exact str with (unpack str)
157162 exact str | (x :: xs)
158163 = foldl SeqEmpty (is x) (map is xs)
159164
160−||| Recognise a whitespace character
165+||| Recognise a single hexidecimal digit
166+export
167+hexDigit : Lexer
168+hexDigit = digit <|> oneOf "abcdefABCDEF"
169+
170+||| Recognise one or more hexidecimal digits
171+export
172+hexDigits : Lexer
173+hexDigits = some hexDigit
174+
175+||| Recognise a single alpha character
176+export
177+alpha : Lexer
178+alpha = pred isAlpha
179+
180+||| Recognise one or more alpha characters
181+export
182+alphas : Lexer
183+alphas = some alpha
184+
185+||| Recognise a lowercase alpha character
186+export
187+lower : Lexer
188+lower = pred isLower
189+
190+||| Recognise one or more lowercase alpha characters
191+export
192+lowers : Lexer
193+lowers = some lower
194+
195+||| Recognise an uppercase alpha character
196+export
197+upper : Lexer
198+upper = pred isUpper
199+
200+||| Recognise one or more uppercase alpha characters
201+export
202+uppers : Lexer
203+uppers = some upper
204+
205+||| Recognise an alphanumeric character
206+export
207+alphaNum : Lexer
208+alphaNum = pred isAlphaNum
209+
210+||| Recognise one or more alphanumeric characters
211+export
212+alphaNums : Lexer
213+alphaNums = some alphaNum
214+
215+||| Recognise a single whitespace character
161216export
162217space : Lexer
163−space = some (pred isSpace)
218+space = pred isSpace
219+
220+||| Recognise one or more whitespace characters
221+export
222+spaces : Lexer
223+spaces = some space
164224
165−||| Recognise a non-alphanumeric, non-whitespace character
225+||| Recognise a single non-whitespace, non-alphanumeric character
166226export
167227symbol : Lexer
168−symbol = some (pred (\x => not (isAlphaNum x) && not (isSpace x)))
228+symbol = pred (\x => not (isSpace x || isAlphaNum x))
229+
230+||| Recognise one or more non-whitespace, non-alphanumeric characters
231+export
232+symbols : Lexer
233+symbols = some symbol
169234
170235||| Recognise zero or more occurrences of a sub-lexer between
171236||| delimiting lexers
@@ -202,6 +267,11 @@ export
202267intLit : Lexer
203268intLit = opt (is '-') <+> digits
204269
270+||| Recognise a hexidecimal literal, prefixed by "0x" or "0X"
271+export
272+hexLit : Lexer
273+hexLit = is '0' <+> oneOf "xX" <+> hexDigits
274+
205275||| A mapping from lexers to the tokens they produce.
206276||| This is a list of pairs `(Lexer, String -> tokenType)`
207277||| For each Lexer in the list, if a substring in the input matches, run