git.iko.soy/typeable/pandoc
on master help sign in
Merge pull request #1363 from jkr/newNormalize
cab1602John MacFarlane
src/Text/Pandoc/Readers/Docx.hs+58−12
@@ -148,25 +148,71 @@ runElemsToString = concatMap runElemToString
148--- Text.Pandoc.Shared.normalize for reasons of efficiency. For
148--- Text.Pandoc.Shared.normalize for reasons of efficiency. For
149--- whatever reason, `normalize` makes a run take almost twice as
149--- whatever reason, `normalize` makes a run take almost twice as
150--- long. (It does more, but this does what we need)
150--- long. (It does more, but this does what we need)
151−strNormalize :: [Inline] -> [Inline]
151+inlineNormalize :: [Inline] -> [Inline]
152−strNormalize [] = []
152+inlineNormalize [] = []
153−strNormalize (Str "" : ils) = strNormalize ils
153+inlineNormalize (Str "" : ils) = inlineNormalize ils
154−strNormalize ((Str s) : (Str s') : l) = strNormalize ((Str (s++s')) : l)
154+inlineNormalize ((Str s) : (Str s') : l) =
155−strNormalize (il:ils) = il : (strNormalize ils)
155+ inlineNormalize (Str (s++s') : l)
156+inlineNormalize ((Emph ils) : (Emph ils') : l) =
157+ inlineNormalize $ (Emph $ inlineNormalize (ils ++ ils')) : l
158+inlineNormalize ((Emph ils) : l) =
159+ Emph (inlineNormalize ils) : (inlineNormalize l)
160+inlineNormalize ((Strong ils) : (Strong ils') : l) =
161+ inlineNormalize $ (Strong $ inlineNormalize (ils ++ ils')) : l
162+inlineNormalize ((Strong ils) : l) =
163+ Strong (inlineNormalize ils) : (inlineNormalize l)
164+inlineNormalize ((Strikeout ils) : (Strikeout ils') : l) =
165+ inlineNormalize $ (Strikeout $ inlineNormalize (ils ++ ils')) : l
166+inlineNormalize ((Strikeout ils) : l) =
167+ Strikeout (inlineNormalize ils) : (inlineNormalize l)
168+inlineNormalize ((Superscript ils) : (Superscript ils') : l) =
169+ inlineNormalize $ (Superscript $ inlineNormalize (ils ++ ils')) : l
170+inlineNormalize ((Superscript ils) : l) =
171+ Superscript (inlineNormalize ils) : (inlineNormalize l)
172+inlineNormalize ((Subscript ils) : (Subscript ils') : l) =
173+ inlineNormalize $ (Subscript $ inlineNormalize (ils ++ ils')) : l
174+inlineNormalize ((Subscript ils) : l) =
175+ Subscript (inlineNormalize ils) : (inlineNormalize l)
176+inlineNormalize ((Space : Space : l)) =
177+ inlineNormalize $ (Space : l)
178+inlineNormalize ((Quoted qt ils) : l) =
179+ Quoted qt (inlineNormalize ils) : inlineNormalize l
180+inlineNormalize ((Cite cits ils) : l) =
181+ let
182+ f :: Citation -> Citation
183+ f (Citation s pref suff mode num hash) =
184+ Citation s (inlineNormalize pref) (inlineNormalize suff) mode num hash
185+ in
186+ Cite (map f cits) (inlineNormalize ils) : (inlineNormalize l)
187+inlineNormalize ((Link ils s) : l) =
188+ Link (inlineNormalize ils) s : (inlineNormalize l)
189+inlineNormalize ((Image ils s) : l) =
190+ Image (inlineNormalize ils) s : (inlineNormalize l)
191+inlineNormalize ((Note blks) : l) =
192+ Note (map blockNormalize blks) : (inlineNormalize l)
193+inlineNormalize ((Span attr ils) : l) =
194+ Span attr (inlineNormalize ils) : (inlineNormalize l)
195+inlineNormalize (il : l) = il : (inlineNormalize l)
156
196
157stripSpaces :: [Inline] -> [Inline]
197stripSpaces :: [Inline] -> [Inline]
158stripSpaces ils =
198stripSpaces ils =
159 reverse $ dropWhile (Space ==) $ reverse $ dropWhile (Space ==) ils
199 reverse $ dropWhile (Space ==) $ reverse $ dropWhile (Space ==) ils
160
200
161blockNormalize :: Block -> Block
201blockNormalize :: Block -> Block
162−blockNormalize (Plain ils) = Plain $ bottomUp strNormalize $ stripSpaces ils
202+blockNormalize (Plain ils) = Plain $ stripSpaces $ inlineNormalize ils
163−blockNormalize (Para ils) = Para $ bottomUp strNormalize $ stripSpaces ils
203+blockNormalize (Para ils) = Para $ stripSpaces $ inlineNormalize ils
164blockNormalize (Header n attr ils) =
204blockNormalize (Header n attr ils) =
165− Header n attr $ bottomUp strNormalize $ stripSpaces ils
205+ Header n attr $ stripSpaces $ inlineNormalize ils
166blockNormalize (Table ils align width hdr cells) =
206blockNormalize (Table ils align width hdr cells) =
167− Table (bottomUp strNormalize $ stripSpaces ils) align width hdr cells
207+ Table (stripSpaces $ inlineNormalize ils) align width hdr cells
168blockNormalize (DefinitionList pairs) =
208blockNormalize (DefinitionList pairs) =
169− DefinitionList $ map (\(ils, blklsts) -> (bottomUp strNormalize (stripSpaces ils), blklsts)) pairs
209+ DefinitionList $ map (\(ils, blklsts) -> (stripSpaces (inlineNormalize ils), (map (map blockNormalize) blklsts))) pairs
210+blockNormalize (BlockQuote blks) = BlockQuote (map blockNormalize blks)
211+blockNormalize (OrderedList attr blkslst) =
212+ OrderedList attr $ map (\blks -> map blockNormalize blks) blkslst
213+blockNormalize (BulletList blkslst) =
214+ BulletList $ map (\blks -> map blockNormalize blks) blkslst
215+blockNormalize (Div attr blks) = Div attr (map blockNormalize blks)
170blockNormalize blk = blk
216blockNormalize blk = blk
171
217
172runToInlines :: ReaderOptions -> Docx -> Run -> [Inline]
218runToInlines :: ReaderOptions -> Docx -> Run -> [Inline]
@@ -240,8 +286,8 @@ parPartsToInlines opts docx parparts =
240 (if False -- TODO depend on option
286 (if False -- TODO depend on option
241 then bottomUp (makeImagesSelfContained docx)
287 then bottomUp (makeImagesSelfContained docx)
242 else id) $
288 else id) $
243− bottomUp spanCorrect $
244 bottomUp spanTrim $
289 bottomUp spanTrim $
290+ bottomUp spanCorrect $
245 bottomUp spanReduce $
291 bottomUp spanReduce $
246 concatMap (parPartToInlines opts docx) parparts
292 concatMap (parPartToInlines opts docx) parparts
247
293
@@ -315,7 +361,7 @@ makeImagesSelfContained _ inline = inline
315bodyToBlocks :: ReaderOptions -> Docx -> Body -> [Block]
361bodyToBlocks :: ReaderOptions -> Docx -> Body -> [Block]
316bodyToBlocks opts docx (Body bps) =
362bodyToBlocks opts docx (Body bps) =
317 bottomUp removeEmptyPars $
363 bottomUp removeEmptyPars $
318− bottomUp blockNormalize $
364+ map blockNormalize $
319 bottomUp spanRemove $
365 bottomUp spanRemove $
320 bottomUp divRemove $
366 bottomUp divRemove $
321 map (makeHeaderAnchors) $
367 map (makeHeaderAnchors) $
tests/Tests/Readers/Docx.hs+8
@@ -74,6 +74,14 @@ tests = [ testGroup "inlines"
74 "literal tabs"
74 "literal tabs"
75 "docx.tabs.docx"
75 "docx.tabs.docx"
76 "docx.tabs.native"
76 "docx.tabs.native"
77+ , testCompare
78+ "normalizing inlines"
79+ "docx.normalize.docx"
80+ "docx.normalize.native"
81+ , testCompare
82+ "normalizing inlines deep inside blocks"
83+ "docx.deep_normalize.docx"
84+ "docx.deep_normalize.native"
77 ]
85 ]
78 , testGroup "blocks"
86 , testGroup "blocks"
79 [ testCompare
87 [ testCompare
tests/docx.deep_normalize.docxnew file

Binary file not shown.

tests/docx.deep_normalize.nativenew file+6
@@ -0,0 +1,6 @@
1+[OrderedList (1,Decimal,OneParen)
2+ [[Para [Str "This",Space,Str "is",Space,Str "at",Space,Str "the",Space,Str "first",Space,Str "level"]
3+ ,OrderedList (1,LowerAlpha,DefaultDelim)
4+ [[Para [Str "This",Space,Str "is",Space,Str "at",Space,Str "the",Space,Str "second",Space,Str "level"]
5+ ,OrderedList (1,LowerRoman,DefaultDelim)
6+ [[Para [Str "This",Space,Str "is",Space,Emph [Str "at",Space,Strong [Str "the",Space,Str "third",Space,Str "level"],Str ",",Space,Str "and",Space,Str "I",Space,Str "want",Space,Str "to"],Space,Str "test",Space,Str "normalization",Space,Str "here."]]]]]]]]
tests/docx.normalize.docxnew file

Binary file not shown.

tests/docx.normalize.nativenew file+2
@@ -0,0 +1,2 @@
1+[Para [Str "These",Space,Str "are",Space,Str "different",Space,Str "fonts."]
2+,Para [Strong [Str "These",Space,Emph [Str "are",Space,Strikeout [Str "different"]],Space,Str "fonts."]]]