2415b26 · typeable/pandoc

git.iko.soy/typeable/pandoc

Test suite: a more robust way of testing the executable.

Mmny of our tests require running the pandoc executable. This is problematic for a few different reasons. First, cabal-install will sometimes run the test suite after building the library but before building the executable, which means the executable isn't in place for the tests. One can work around that by first building, then building and running the tests, but that's fragile. Second, we have to find the executable. So far, we've done that using a function findPandoc that attempts to locate it relative to the test executable (which can be located using findExecutablePath). But the logic here is delicate and work with every combination of options. To solve both problems, we add an `--emulate` option to the `test-pandoc` executable. When `--emulate` occurs as the first argument passed to `test-pandoc`, the program simply emulates the regular pandoc executable, using the rest of the arguments (after `--emulate`). Thus, test-pandoc --emulate -f markdown -t latex is just like pandoc -f markdown -t latex Since all the work is done by library functions, implementing this emulation just takes a couple lines of code and should be entirely reliable. With this change, we can test the pandoc executable by running the test program itself (locatable using findExecutablePath) with the `--emulate` option. This removes the need for the fragile `findPandoc` step, and it means we can run our integration tests even when we're just building the library, not the executable. Part of this change involved simplifying some complex handling to set environment variables for dynamic library paths. I have tested a build with `--enable-dynamic-executable`, and it works, but further testing may be needed.

commit2415b2680a522e89b63abb370c02bfff54b824a2 authorJohn MacFarlane <jgm@berkeley.edu> dateTue 2 Feb 2021, 17:09 (6 years ago) parentec85092

4 files changed+64−72

compare with master

unifiedsplit

test/Tests/Command.hs+33−28 test/Tests/Helpers.hs−29 test/Tests/Old.hs+12−9 test/test-pandoc.hs+19−6

test/Tests/Command.hs+33−28

@@ -1,4 +1,5 @@

11{-# LANGUAGE NoImplicitPrelude #-}

2+{-# LANGUAGE TupleSections #-}

23{- |

34 Module : Tests.Command

@@ -10,11 +11,12 @@

1011

1112Run commands, and test results, defined in markdown files.

1213-}

13−module Tests.Command (findPandoc, runTest, tests)

14+module Tests.Command (runTest, tests)

1415where

1516

1617import Prelude

1718import Data.Algorithm.Diff

19+import System.Environment.Executable (getExecutablePath)

1820import qualified Data.ByteString as BS

1921import qualified Data.Text as T

2022import Data.List (isSuffixOf, intercalate)

@@ -34,27 +36,21 @@ import Text.Pandoc

3436import qualified Text.Pandoc.UTF8 as UTF8

3537

3638-- | Run a test with and return output.

37−execTest :: FilePath -- ^ Path to pandoc

39+execTest :: String -- ^ Path to test executable

3840 -> String -- ^ Shell command

3941 -> String -- ^ Input text

4042 -> IO (ExitCode, String) -- ^ Exit code and actual output

41−execTest pandocpath cmd inp = do

43+execTest testExePath cmd inp = do

4244 mldpath <- Env.lookupEnv "LD_LIBRARY_PATH"

4345 mdyldpath <- Env.lookupEnv "DYLD_LIBRARY_PATH"

44− let findDynlibDir [] = Nothing

45− findDynlibDir ("build":xs) = Just $ joinPath (reverse xs) </> "build"

46− findDynlibDir (_:xs) = findDynlibDir xs

47− let mbDynlibDir = findDynlibDir (reverse $ splitDirectories $

48− takeDirectory $ takeWhile (/=' ') cmd)

49− let dynlibEnv = [("DYLD_LIBRARY_PATH",

50− intercalate ":" $ catMaybes [mbDynlibDir, mdyldpath])

51− ,("LD_LIBRARY_PATH",

52− intercalate ":" $ catMaybes [mbDynlibDir, mldpath])]

53− let env' = dynlibEnv ++ [("PATH",takeDirectory pandocpath),("TMP","."),

54− ("LANG","en_US.UTF-8"),

55− ("HOME", "./"),

56− ("pandoc_datadir", "..")]

57− let pr = (shell cmd){ env = Just env' }

46+ let env' = ("PATH",takeDirectory testExePath) :

47+ ("TMP",".") :

48+ ("LANG","en_US.UTF-8") :

49+ ("HOME", "./") :

50+ ("pandoc_datadir", "..") :

51+ maybe [] ((:[]) . ("LD_LIBRARY_PATH",)) mldpath ++

52+ maybe [] ((:[]) . ("DYLD_LIBRARY_PATH",)) mdyldpath

53+ let pr = (shell (pandocToEmulate True cmd)){ env = Just env' }

5854 (ec, out', err') <- readCreateProcessWithExitCode pr inp

5955 -- filter \r so the tests will work on Windows machines

6056 let out = filter (/= '\r') $ err' ++ out'

@@ -63,15 +59,23 @@ execTest pandocpath cmd inp = do

6359 ExitSuccess -> return ()

6460 return (ec, out)

6561

62+pandocToEmulate :: Bool -> String -> String

63+pandocToEmulate True ('p':'a':'n':'d':'o':'c':cs) =

64+ "test-pandoc --emulate" ++ pandocToEmulate False cs

65+pandocToEmulate False ('|':' ':'p':'a':'n':'d':'o':'c':cs) =

66+ "| " ++ "test-pandoc --emulate" ++ pandocToEmulate False cs

67+pandocToEmulate _ (c:cs) = c : pandocToEmulate False cs

68+pandocToEmulate _ [] = []

69+

6670-- | Run a test, return True if test passed.

67−runTest :: String -- ^ Title of test

68− -> FilePath -- ^ Path to pandoc

71+runTest :: String -- ^ Path to test executable

72+ -> String -- ^ Title of test

6973 -> String -- ^ Shell command

7074 -> String -- ^ Input text

7175 -> String -- ^ Expected output

7276 -> TestTree

73−runTest testname pandocpath cmd inp norm = testCase testname $ do

74− (ec, out) <- execTest pandocpath cmd inp

77+runTest testExePath testname cmd inp norm = testCase testname $ do

78+ (ec, out) <- execTest testExePath cmd inp

7579 result <- if ec == ExitSuccess

7680 then

7781 if out == norm

@@ -82,12 +86,13 @@ runTest testname pandocpath cmd inp norm = testCase testname $ do

8286 else return $ TestError ec

8387 assertBool (show result) (result == TestPassed)

8488

85−tests :: FilePath -> TestTree

89+tests :: TestTree

8690{-# NOINLINE tests #-}

87−tests pandocPath = unsafePerformIO $ do

91+tests = unsafePerformIO $ do

8892 files <- filter (".md" `isSuffixOf`) <$>

8993 getDirectoryContents "command"

90− let cmds = map (extractCommandTest pandocPath) files

94+ testExePath <- getExecutablePath

95+ let cmds = map (extractCommandTest testExePath) files

9196 return $ testGroup "Command:" cmds

9297

9398isCodeBlock :: Block -> Bool

@@ -103,7 +108,7 @@ dropPercent ('%':xs) = dropWhile (== ' ') xs

103108dropPercent xs = xs

104109

105110runCommandTest :: FilePath -> FilePath -> Int -> String -> TestTree

106−runCommandTest pandocpath fp num code =

111+runCommandTest testExePath fp num code =

107112 goldenTest testname getExpected getActual compareValues updateGolden

108113 where

109114 testname = "#" <> show num

@@ -116,7 +121,7 @@ runCommandTest pandocpath fp num code =

116121 input = unlines inplines

117122 norm = unlines normlines

118123 getExpected = return norm

119− getActual = snd <$> execTest pandocpath cmd input

124+ getActual = snd <$> execTest testExePath cmd input

120125 compareValues expected actual

121126 | actual == expected = return Nothing

122127 | otherwise = return $ Just $ "--- test/command/" ++ fp ++ "\n+++ " ++

@@ -132,10 +137,10 @@ runCommandTest pandocpath fp num code =

132137 UTF8.writeFile fp' updated

133138

134139extractCommandTest :: FilePath -> FilePath -> TestTree

135−extractCommandTest pandocpath fp = unsafePerformIO $ do

140+extractCommandTest testExePath fp = unsafePerformIO $ do

136141 contents <- UTF8.toText <$> BS.readFile ("command" </> fp)

137142 Pandoc _ blocks <- runIOorExplode (readMarkdown

138143 def{ readerExtensions = pandocExtensions } contents)

139144 let codeblocks = map extractCode $ filter isCodeBlock blocks

140− let cases = zipWith (runCommandTest pandocpath fp) [1..] codeblocks

145+ let cases = zipWith (runCommandTest testExePath fp) [1..] codeblocks

141146 return $ testGroup fp cases

test/Tests/Helpers.hs−29

@@ -15,7 +15,6 @@ Utility functions for the test suite.

1515module Tests.Helpers ( test

1616 , TestResult(..)

1717 , showDiff

18− , findPandoc

1918 , (=?>)

2019 , purely

2120 , ToString(..)

@@ -86,34 +85,6 @@ showDiff (l,r) (Second ln : ds) =

8685showDiff (l,r) (Both _ _ : ds) =

8786 showDiff (l+1,r+1) ds

8887

89−-- | Find pandoc executable relative to test-pandoc

90−findPandoc :: IO FilePath

91−findPandoc = do

92− testExePath <- getExecutablePath

93− let pandocDir =

94− case reverse (splitDirectories (takeDirectory testExePath)) of

95− -- cabalv2 with --disable-optimization

96− "test-pandoc" : "build" : "noopt" : "test-pandoc" : "t" : ps

97− -> joinPath (reverse ps) </>

98− "x" </> "pandoc" </> "noopt" </> "build" </> "pandoc"

99− -- cabalv2 without --disable-optimization

100− "test-pandoc" : "build" : "test-pandoc" : "t" : ps

101− -> joinPath (reverse ps) </>

102− "x" </> "pandoc" </> "build" </> "pandoc"

103− -- cabalv1

104− "test-pandoc" : "build" : ps

105− -> joinPath (reverse ps) </> "build" </> "pandoc"

106− _ -> error "findPandoc: could not find pandoc executable"

107− let pandocPath = pandocDir </> "pandoc"

108−#ifdef _WINDOWS

109− <.> "exe"

110−#endif

111− found <- doesFileExist pandocPath

112− if found

113− then return pandocPath

114− else error $ "findPandoc: could not find pandoc executable at "

115− ++ pandocPath

116−

11788vividize :: Diff String -> String

11889vividize (Both s _) = " " ++ s

11990vividize (First s) = "- " ++ s

test/Tests/Old.hs+12−9

@@ -1,4 +1,5 @@

11{-# LANGUAGE NoImplicitPrelude #-}

2+{-# LANGUAGE TupleSections #-}

23{- |

34 Module : Tests.Old

@@ -19,6 +20,7 @@ import Data.Maybe (catMaybes)

1920import System.Exit

2021import System.FilePath (joinPath, splitDirectories, (<.>), (</>))

2122import qualified System.Environment as Env

23+import System.Environment.Executable (getExecutablePath)

2224import Text.Pandoc.Process (pipeProcess)

2325import Test.Tasty (TestTree, testGroup)

2426import Test.Tasty.Golden.Advanced (goldenTest)

@@ -322,13 +324,14 @@ testWithNormalize normalizer pandocPath testname opts inp norm =

322324 getActual = do

323325 mldpath <- Env.lookupEnv "LD_LIBRARY_PATH"

324326 mdyldpath <- Env.lookupEnv "DYLD_LIBRARY_PATH"

325− let mbDynlibDir = findDynlibDir (reverse $

326− splitDirectories pandocPath)

327− let dynlibEnv = [("DYLD_LIBRARY_PATH", intercalate ":" $ catMaybes [mbDynlibDir, mdyldpath])

328− ,("LD_LIBRARY_PATH", intercalate ":" $ catMaybes [mbDynlibDir, mldpath])]

329− let env = dynlibEnv ++

330− [("TMP","."),("LANG","en_US.UTF-8"),("HOME", "./")]

331− (ec, out) <- pipeProcess (Just env) pandocPath options mempty

327+ let env = ("TMP",".") :

328+ ("LANG","en_US.UTF-8") :

329+ ("HOME", "./") :

330+ maybe [] ((:[]) . ("LD_LIBRARY_PATH",)) mldpath ++

331+ maybe [] ((:[]) . ("DYLD_LIBRARY_PATH",)) mdyldpath

332+

333+ (ec, out) <- pipeProcess (Just env) pandocPath

334+ ("--emulate":options) mempty

332335 if ec == ExitSuccess

333336 then return $ filter (/='\r') . normalizer

334337 $ UTF8.toStringLazy out

@@ -339,8 +342,8 @@ testWithNormalize normalizer pandocPath testname opts inp norm =

339342

340343compareValues :: FilePath -> [String] -> String -> String -> IO (Maybe String)

341344compareValues norm options expected actual = do

342− pandocPath <- findPandoc

343− let cmd = pandocPath ++ " " ++ unwords options

345+ testExePath <- getExecutablePath

346+ let cmd = testExePath ++ " --emulate " ++ unwords options

344347 let dash = replicate 72 '-'

345348 let diff = getDiff (lines actual) (lines expected)

346349 if expected == actual

test/test-pandoc.hs+19−6

@@ -4,6 +4,12 @@

44module Main where

66import Prelude

7+import System.Environment (getArgs)

8+import qualified Control.Exception as E

9+import Text.Pandoc.App (convertWithOpts, defaultOpts, options,

10+ parseOptionsFromArgs)

11+import Text.Pandoc.Error (handleError)

12+import System.Environment.Executable (getExecutablePath)

713import GHC.IO.Encoding

814import Test.Tasty

915import qualified Tests.Command

@@ -46,12 +52,11 @@ import qualified Tests.Writers.Powerpoint

4652import qualified Tests.Writers.RST

4753import qualified Tests.Writers.AnnotatedTable

4854import qualified Tests.Writers.TEI

49−import Tests.Helpers (findPandoc)

5055import Text.Pandoc.Shared (inDirectory)

5156

5257tests :: FilePath -> TestTree

5358tests pandocPath = testGroup "pandoc tests"

54− [ Tests.Command.tests pandocPath

59+ [ Tests.Command.tests

5560 , testGroup "Old" (Tests.Old.tests pandocPath)

5661 , testGroup "Shared" Tests.Shared.tests

5762 , testGroup "Writers"

@@ -102,7 +107,15 @@ tests pandocPath = testGroup "pandoc tests"

102107main :: IO ()

103108main = do

104109 setLocaleEncoding utf8

105− inDirectory "test" $ do

106− fp <- findPandoc

107− putStrLn $ "Using pandoc executable at " ++ fp

108− defaultMain $ tests fp

110+ args <- getArgs

111+ case args of

112+ "--emulate":args' -> -- emulate pandoc executable

113+ E.catch

114+ (parseOptionsFromArgs options defaultOpts "pandoc" args' >>=

115+ convertWithOpts)

116+ (handleError . Left)

117+ _ -> inDirectory "test" $ do

118+ fp <- getExecutablePath

119+ -- putStrLn $ "Using pandoc executable at " ++ fp

120+ defaultMain $ tests fp

121+