7e38b8e · typeable/pandoc

src/Text/Pandoc/Readers/LaTeX.hs−2

compare with master

unifiedsplit

@@ -22,8 +22,6 @@ module Text.Pandoc.Readers.LaTeX ( readLaTeX,

22 rawLaTeXInline,

23 rawLaTeXBlock,

24 inlineCommand,

25− tokenize,

26− untokenize

27 ) where

25 ) where

28

26

29import Control.Applicative (many, optional, (<|>))

27import Control.Applicative (many, optional, (<|>))

src/Text/Pandoc/Readers/LaTeX/Parsing.hs+9

@@ -292,6 +292,15 @@ applyMacros s = (guardDisabled Ext_latex_macros >> return s) <|>

292 Left e -> Prelude.fail (show e)

293 Right s' -> return s'

294

295+{-

296+When tokenize or untokenize change, test with this

297+QuickCheck property:

298+

299+> tokUntokRoundtrip :: String -> Bool

300+> tokUntokRoundtrip s =

301+> let t = T.pack s in untokenize (tokenize "random" t) == t

302+-}

303+

295tokenize :: SourceName -> Text -> [Tok]

304tokenize :: SourceName -> Text -> [Tok]

296tokenize sourcename = totoks (initialPos sourcename)

305tokenize sourcename = totoks (initialPos sourcename)

297

306

test/Tests/Readers/LaTeX.hs+1−16

@@ -15,10 +15,8 @@ module Tests.Readers.LaTeX (tests) where

15import Data.Text (Text)

16import qualified Data.Text as T

17import qualified Text.Pandoc.UTF8 as UTF8

18−import Text.Pandoc.Readers.LaTeX (tokenize, untokenize)

19import Test.Tasty

18import Test.Tasty

20import Test.Tasty.HUnit

19import Test.Tasty.HUnit

21−import Test.Tasty.QuickCheck

22import Tests.Helpers

20import Tests.Helpers

23import Text.Pandoc

21import Text.Pandoc

24import Text.Pandoc.Arbitrary ()

22import Text.Pandoc.Arbitrary ()

@@ -47,21 +45,8 @@ simpleTable' aligns rows

47 where

45 where

48 toRow = Row nullAttr . map simpleCell

46 toRow = Row nullAttr . map simpleCell

49

47

50−tokUntokRt :: String -> Bool

51−tokUntokRt s = untokenize (tokenize "random" t) == t

52− where t = T.pack s

53−

54tests :: [TestTree]

48tests :: [TestTree]

55−tests = [ testGroup "tokenization"

49+tests = [ testGroup "basic"

56− [ testCase "tokenizer round trip on test case" $ do

57− orig <- UTF8.readFile "../test/latex-reader.latex"

58− let new = untokenize $ tokenize "../test/latex-reader.latex"

59− orig

60− assertEqual "untokenize . tokenize is identity" orig new

61− , testProperty "untokenize . tokenize is identity" tokUntokRt

62− ]

63−

64− , testGroup "basic"

65 [ "simple" =:

50 [ "simple" =:

66 "word" =?> para "word"

51 "word" =?> para "word"

67 , "space" =:

52 , "space" =: