kanaka/mal
1
1
Fork 0
mirror of https://github.com/kanaka/mal.git synced 2024-11-10 12:47:45 +03:00
Code Issues Projects Releases Wiki Activity

Use compiled regex instead of raw ones

Browse Source
This commit is contained in:
Iqbal Ansari 2016-08-28 10:45:58 +05:30
parent 961c035bc3
commit e929d5be8e
1 changed files with 15 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

29
clisp/reader.lisp
View File

@ -12,9 +12,9 @@
\\)\\|[^\"\\]\\)*\"$"
"RE string")
(defvar *tokenizer-re* "[[:space:],]*\\(~@\\|[][{}()~`'^@]\\|\"\\(\\\\\\(.\\|
(defvar *tokenizer-re* (regexp:regexp-compile "[[:space:],]*\\(~@\\|[][{}()~`'^@]\\|\"\\(\\\\\\(.\\|
\\)\\|[^\"\\]\\)*\"\\?\\|;[^
]*\\|[^][[:space:]~{}()@^`'\";]*\\)"
]*\\|[^][[:space:]~{}()@^`'\";]*\\)")
"RE")
(define-condition eof (types:mal-error)
@ -50,20 +50,21 @@
'(#\Space #\Newline #\Backspace #\Tab
#\Linefeed #\Page #\Return #\Rubout #\,))
(defun tokenize (string)
(let (tokens)
(loop
with end = (length string)
for start = 0 then (regexp:match-end match)
for match = (when (< start end)
(regexp:match *tokenizer-re* string :start start))
while match
do (let ((token (string-trim *whitespace-chars*
(regexp:match-string string match))))
(unless (or (zerop (length token))
(char= (char token 0) #\;))
(push token tokens))))
(do* ((start 0)
(end (length string))
(match t))
((not match))
(setf match (when (< start end)
(nth-value 1
(regexp:regexp-exec *tokenizer-re* string :start start))))
(when match
(setf start (regexp:match-end match))
(let ((token (regexp:match-string string match)))
(unless (or (zerop (length token))
(char= (char token 0) #\;))
(push token tokens)))))
(nreverse tokens)))
(defstruct (token-reader)