@@ -1,12 +1,25 @@
11||| A simple module to process 'literate' documents.
22|||
3−||| The underlying tokeniser is greedy. Once it identifies a line
4−||| marker it reads until the end of line, and once it recognises a
5−||| start block, it will consume until the next identifiable end
6−||| block. Thus, the input literate files must be well-formed w.r.t to
7−||| line markers and code blocks. A further restriction is that
8−||| literate documents cannot contain the markers within the
9−||| document's main text: This will confuse the lexer..
3+||| The module uses a lexer to split the document into code blocks,
4+||| delineated by user-defined markers, and code lines that are
5+||| indicated be a line marker. The lexer returns a document stripped
6+||| of non-code elements but preserving the original document's line
7+||| count. Column numbering of code lines are not preserved.
8+|||
9+||| The underlying tokeniser is greedy.
10+|||
11+||| Once it identifies a line marker it reads a prettifying space then
12+||| consumes until the end of line. Once identifies a starting code
13+||| block marker, the lexer will consume input until the next
14+||| identifiable end block is encountered. Any other content is
15+||| treated as part of the original document.
16+|||
17+||| Thus, the input literate files *must* be well-formed w.r.t
18+||| to code line markers and code blocks.
19+|||
20+||| A further restriction is that literate documents cannot contain
21+||| the markers within the document's main text: This will confuse the
22+||| lexer.
1023|||
1124module Text.Literate
1225
@@ -20,7 +33,7 @@ untilEOL : Recognise False
2033untilEOL = manyTill any (is '\n')
2134
2235line : String -> Lexer
23−line s = exact s <+> untilEOL
36+line s = exact s <+> space <+> untilEOL
2437
2538block : String -> String -> Lexer
2639block s e = exact s <+> manyTill any (exact e)
@@ -45,15 +58,25 @@ rawTokens delims ls =
4558||| Merge the tokens into a single source file.
4659reduce : List (TokenData Token) -> String -> String
4760reduce [] acc = acc
48−reduce (MkToken _ _ (Any x) :: rest) acc = reduce rest acc
49−
50−reduce (MkToken _ _ (CodeLine m src) :: rest) acc = reduce rest (acc ++ (substr (length m) (length src) src))
61+reduce (MkToken _ _ (Any x) :: rest) acc = reduce rest (acc ++ blank_content)
62+ where
63+ -- Preserve the original document's line count.
64+ blank_content : String
65+ blank_content = if elem '\n' (unpack x)
66+ then concat $ replicate (length (lines x)) "\n"
67+ else ""
68+reduce (MkToken _ _ (CodeLine m src) :: rest) acc =
69+ reduce rest (acc ++ (substr
70+ (length m + 1) -- remove space to right of marker.
71+ (length src)
72+ src))
5173
5274reduce (MkToken _ _ (CodeBlock l r src) :: rest) acc with (lines src) -- Strip the deliminators surrounding the block.
5375 reduce (MkToken _ _ (CodeBlock l r src) :: rest) acc | [] = reduce rest acc -- 1
5476 reduce (MkToken _ _ (CodeBlock l r src) :: rest) acc | (s :: ys) with (snocList ys)
5577 reduce (MkToken _ _ (CodeBlock l r src) :: rest) acc | (s :: []) | Empty = reduce rest acc -- 2
56− reduce (MkToken _ _ (CodeBlock l r src) :: rest) acc | (s :: (srcs ++ [f])) | (Snoc rec) = reduce rest (acc ++ unlines srcs)
78+ reduce (MkToken _ _ (CodeBlock l r src) :: rest) acc | (s :: (srcs ++ [f])) | (Snoc rec) =
79+ reduce rest (acc ++ "\n" ++ unlines srcs ++ "\n")
5780
5881-- [ NOTE ] 1 & 2 shouldn't happen as code blocks are well formed i.e. have two deliminators.
5982