@@ -1,12 +1,25 @@
1||| A simple module to process 'literate' documents.
1||| A simple module to process 'literate' documents.
3+||| The module uses a lexer to split the document into code blocks,
4+||| delineated by user-defined markers, and code lines that are
5+||| indicated be a line marker. The lexer returns a document stripped
6+||| of non-code elements but preserving the original document's line
7+||| count. Column numbering of code lines are not preserved.
3−||| The underlying tokeniser is greedy. Once it identifies a line
9+||| The underlying tokeniser is greedy.
4−||| marker it reads until the end of line, and once it recognises a
10+|||
11+||| Once it identifies a line marker it reads a prettifying space then
12+||| consumes until the end of line. Once identifies a starting code
5−||| start block, it will consume until the next identifiable end
13+||| block marker, the lexer will consume input until the next
14+||| identifiable end block is encountered. Any other content is
15+||| treated as part of the original document.
6−||| block. Thus, the input literate files must be well-formed w.r.t to
17+||| Thus, the input literate files *must* be well-formed w.r.t
7−||| line markers and code blocks. A further restriction is that
18+||| to code line markers and code blocks.
8−||| literate documents cannot contain the markers within the
20+||| A further restriction is that literate documents cannot contain
9−||| document's main text: This will confuse the lexer..
21+||| the markers within the document's main text: This will confuse the
11module Text.Literate
24module Text.Literate
@@ -20,7 +33,7 @@ untilEOL : Recognise False
20untilEOL = manyTill any (is '\n')
33untilEOL = manyTill any (is '\n')
22line : String -> Lexer
35line : String -> Lexer
23−line s = exact s <+> untilEOL
36+line s = exact s <+> space <+> untilEOL
25block : String -> String -> Lexer
38block : String -> String -> Lexer
26block s e = exact s <+> manyTill any (exact e)
39block s e = exact s <+> manyTill any (exact e)
@@ -45,15 +58,25 @@ rawTokens delims ls =
45||| Merge the tokens into a single source file.
58||| Merge the tokens into a single source file.
46reduce : List (TokenData Token) -> String -> String
59reduce : List (TokenData Token) -> String -> String
47reduce [] acc = acc
60reduce [] acc = acc
48−reduce (MkToken _ _ (Any x) :: rest) acc = reduce rest acc
61+reduce (MkToken _ _ (Any x) :: rest) acc = reduce rest (acc ++ blank_content)
63+ -- Preserve the original document's line count.
64+ blank_content : String
65+ blank_content = if elem '\n' (unpack x)
66+ then concat $ replicate (length (lines x)) "\n"
50−reduce (MkToken _ _ (CodeLine m src) :: rest) acc = reduce rest (acc ++ (substr (length m) (length src) src))
68+reduce (MkToken _ _ (CodeLine m src) :: rest) acc =
69+ reduce rest (acc ++ (substr
70+ (length m + 1) -- remove space to right of marker.
52reduce (MkToken _ _ (CodeBlock l r src) :: rest) acc with (lines src) -- Strip the deliminators surrounding the block.
74reduce (MkToken _ _ (CodeBlock l r src) :: rest) acc with (lines src) -- Strip the deliminators surrounding the block.
53 reduce (MkToken _ _ (CodeBlock l r src) :: rest) acc | [] = reduce rest acc -- 1
75 reduce (MkToken _ _ (CodeBlock l r src) :: rest) acc | [] = reduce rest acc -- 1
54 reduce (MkToken _ _ (CodeBlock l r src) :: rest) acc | (s :: ys) with (snocList ys)
76 reduce (MkToken _ _ (CodeBlock l r src) :: rest) acc | (s :: ys) with (snocList ys)
55 reduce (MkToken _ _ (CodeBlock l r src) :: rest) acc | (s :: []) | Empty = reduce rest acc -- 2
77 reduce (MkToken _ _ (CodeBlock l r src) :: rest) acc | (s :: []) | Empty = reduce rest acc -- 2
56− reduce (MkToken _ _ (CodeBlock l r src) :: rest) acc | (s :: (srcs ++ [f])) | (Snoc rec) = reduce rest (acc ++ unlines srcs)
78+ reduce (MkToken _ _ (CodeBlock l r src) :: rest) acc | (s :: (srcs ++ [f])) | (Snoc rec) =
79+ reduce rest (acc ++ "\n" ++ unlines srcs ++ "\n")
58-- [ NOTE ] 1 & 2 shouldn't happen as code blocks are well formed i.e. have two deliminators.
81-- [ NOTE ] 1 & 2 shouldn't happen as code blocks are well formed i.e. have two deliminators.