git.iko.soy/typeable/pandoc
on master help sign in
T.P.Readers.LaTeX: Don't export tokenize, untokenize.

[API change] These were only exported for testing, which seems the wrong thing to do. They don't belong in the public API and are not really usable as they are, without access to the Tok type which is not exported. Removed the tokenize/untokenize roundtrip test. We put a quickcheck property in the comments which may be used when this code is touched (if it is).

commit7e38b8e55a49f027b6aaa690d84225b0374eb057 authorJohn MacFarlane <jgm@berkeley.edu> dateSun 28 Feb 2021, 22:51 (6 years ago) parent2463fbf
3 files changed+10−18
compare with master
unifiedsplit
src/Text/Pandoc/Readers/LaTeX.hs−2 src/Text/Pandoc/Readers/LaTeX/Parsing.hs+9 test/Tests/Readers/LaTeX.hs+1−16
src/Text/Pandoc/Readers/LaTeX.hs−2
@@ -22,8 +22,6 @@ module Text.Pandoc.Readers.LaTeX ( readLaTeX,
2222 rawLaTeXInline,
2323 rawLaTeXBlock,
2424 inlineCommand,
25− tokenize,
26− untokenize
2725 ) where
2826
2927import Control.Applicative (many, optional, (<|>))
src/Text/Pandoc/Readers/LaTeX/Parsing.hs+9
@@ -292,6 +292,15 @@ applyMacros s = (guardDisabled Ext_latex_macros >> return s) <|>
292292 Left e -> Prelude.fail (show e)
293293 Right s' -> return s'
294294
295+{-
296+When tokenize or untokenize change, test with this
297+QuickCheck property:
298+
299+> tokUntokRoundtrip :: String -> Bool
300+> tokUntokRoundtrip s =
301+> let t = T.pack s in untokenize (tokenize "random" t) == t
302+-}
303+
295304tokenize :: SourceName -> Text -> [Tok]
296305tokenize sourcename = totoks (initialPos sourcename)
297306
test/Tests/Readers/LaTeX.hs+1−16
@@ -15,10 +15,8 @@ module Tests.Readers.LaTeX (tests) where
1515import Data.Text (Text)
1616import qualified Data.Text as T
1717import qualified Text.Pandoc.UTF8 as UTF8
18−import Text.Pandoc.Readers.LaTeX (tokenize, untokenize)
1918import Test.Tasty
2019import Test.Tasty.HUnit
21−import Test.Tasty.QuickCheck
2220import Tests.Helpers
2321import Text.Pandoc
2422import Text.Pandoc.Arbitrary ()
@@ -47,21 +45,8 @@ simpleTable' aligns rows
4745 where
4846 toRow = Row nullAttr . map simpleCell
4947
50−tokUntokRt :: String -> Bool
51−tokUntokRt s = untokenize (tokenize "random" t) == t
52− where t = T.pack s
53−
5448tests :: [TestTree]
55−tests = [ testGroup "tokenization"
56− [ testCase "tokenizer round trip on test case" $ do
57− orig <- UTF8.readFile "../test/latex-reader.latex"
58− let new = untokenize $ tokenize "../test/latex-reader.latex"
59− orig
60− assertEqual "untokenize . tokenize is identity" orig new
61− , testProperty "untokenize . tokenize is identity" tokUntokRt
62− ]
63−
64− , testGroup "basic"
49+tests = [ testGroup "basic"
6550 [ "simple" =:
6651 "word" =?> para "word"
6752 , "space" =: