on master help sign in
Parsing via iteratee package
commit5c6adc4d3ff485bc55c8aa5ce29f38ac29a6ff88 authorMichael Snoyman <michael@snoyman.com> dateTue 11 May 2010, 11:53 (16 years ago) parent9e64c3e
3 files changed+201−147
Text/Libyaml.hs+81−42 runtests.hs+102−102 yaml.cabal+18−3
Text/Libyaml.hs+81−42
@@ -23,7 +23,9 @@ module Text.Libyaml
2323 -- * Encoder
2424 , YamlEncoder
2525 , YamlDecoder
26+ , Parser
2627 , parseEvent
28+ , parserParseOne
2729 , emitEvent
2830 -- ** Combinators
2931 , emitStream
@@ -59,6 +61,10 @@ import Control.Monad.Trans.Reader
5961
6062import Control.Exception (throwIO, Exception, SomeException)
6163import Data.Typeable (Typeable)
64+import Data.Iteratee
65+import qualified Data.Iteratee.Base.StreamChunk as SC
66+import Control.Applicative
67+import Control.Monad.CatchIO (MonadCatchIO, finally)
6268
6369data Event =
6470 EventNone
@@ -205,40 +211,11 @@ allocaBytesR i = with (allocaBytes i)
205211withCStringR :: With m => String -> (Ptr CChar -> m a) -> m a
206212withCStringR s = with (withCString s)
207213
208−withFileParser :: (MonadFailure YamlException m, With m)
209− => FilePath
210− -> (Parser -> m a)
211− -> m a
212−withFileParser fp f = allocaBytesR parserSize $ \p -> do
213− res <- liftIO $ c_yaml_parser_initialize p
214− when (res == 0) $ failure YamlOutOfMemory
215− file <- withCStringR fp $ \fp' -> withCStringR "r" $ \r' ->
216− liftIO (c_fopen fp' r')
217− when (file == nullPtr) $ failure $ YamlFileNotFound fp
218− liftIO $ c_yaml_parser_set_input_file p file
219− ret <- f p
220− liftIO $ c_fclose file
221− liftIO $ c_yaml_parser_delete p -- could use some finallys here
222− return ret
223−
224−withParser :: (MonadFailure YamlException m, With m)
225− => B.ByteString
226− -> (Parser -> m a)
227− -> m a
228−withParser bs f = allocaBytesR parserSize $ \p -> do
229− res <- liftIO $ c_yaml_parser_initialize p
230− when (res == 0) $ failure YamlOutOfMemory
231− let (fptr, offset, len) = B.toForeignPtr bs
232− ret <- withForeignPtrR fptr $ \ptr -> do
233− let ptr' = castPtr ptr `plusPtr` offset
234− len' = fromIntegral len
235− liftIO $ c_yaml_parser_set_input_string p ptr' len'
236− f p
237− liftIO $ c_yaml_parser_delete p -- FIXME use finally
238− return ret
239−
240−withForeignPtrR :: With m => ForeignPtr a -> (Ptr a -> m b) -> m b
241−withForeignPtrR fp = with (withForeignPtr fp)
214+withForeignPtr' :: MonadIO m => ForeignPtr a -> (Ptr a -> m b) -> m b
215+withForeignPtr' fp f = do
216+ r <- f $ unsafeForeignPtrToPtr fp
217+ liftIO $ touchForeignPtr fp
218+ return r
242219
243220foreign import ccall unsafe "yaml_parser_parse"
244221 c_yaml_parser_parse :: Parser -> EventRaw -> IO CInt
@@ -659,14 +636,76 @@ encodeFile :: (With m, MonadFailure YamlException m)
659636 -> m ()
660637encodeFile filePath = withEmitterFile filePath . runReaderT
661638
662−decode :: (With m, MonadFailure YamlException m)
639+decode :: SC.StreamChunk c Event
640+ => MonadCatchIO m
663641 => B.ByteString
664− -> YamlDecoder m a
665− -> m a
666−decode bs dec = withParser bs $ runReaderT dec
642+ -> EnumeratorGM c Event m a
643+decode bs i = do
644+ fp <- liftIO $ mallocForeignPtrBytes parserSize
645+ res <- liftIO $ withForeignPtr fp c_yaml_parser_initialize
646+ flip finally (liftIO $ withForeignPtr fp c_yaml_parser_delete) $
647+ if (res == 0)
648+ then return $ throwErr $ Err "Yaml out of memory"
649+ else do
650+ let (fptr, offset, len) = B.toForeignPtr bs
651+ withForeignPtr' fptr $ \ptr -> do
652+ let ptr' = castPtr ptr `plusPtr` offset
653+ len' = fromIntegral len
654+ liftIO $ withForeignPtr fp $ \p ->
655+ c_yaml_parser_set_input_string p ptr' len'
656+ runParser fp i
667657
668−decodeFile :: (With m, MonadFailure YamlException m)
658+decodeFile :: SC.StreamChunk c Event
659+ => MonadCatchIO m
669660 => FilePath
670− -> YamlDecoder m a
671− -> m a
672−decodeFile fp dec = withFileParser fp $ runReaderT dec
661+ -> EnumeratorGM c Event m a
662+decodeFile file i = do
663+ fp <- liftIO $ mallocForeignPtrBytes parserSize
664+ res <- liftIO $ withForeignPtr fp c_yaml_parser_initialize
665+ flip finally (liftIO $ withForeignPtr fp c_yaml_parser_delete) $
666+ if (res == 0)
667+ then return $ throwErr $ Err "Yaml out of memory"
668+ else do
669+ file' <- liftIO
670+ $ withCStringR file $ \file' -> withCStringR "r" $ \r' ->
671+ c_fopen file' r'
672+ if (file' == nullPtr)
673+ then return $ throwErr $ Err
674+ $ "Yaml file not found: " ++ file
675+ else do
676+ liftIO $ withForeignPtr fp $ \p ->
677+ c_yaml_parser_set_input_file p file'
678+ finally (runParser fp i) $ liftIO $ do
679+ c_fclose file'
680+ withForeignPtr fp c_yaml_parser_delete
681+
682+runParser :: SC.StreamChunk c Event
683+ => MonadCatchIO m
684+ => ForeignPtr ParserStruct
685+ -> IterateeG c Event m a
686+ -> m (IterateeG c Event m a)
687+runParser fp iter = do
688+ e <- liftIO $ withForeignPtr fp parserParseOne'
689+ case e of
690+ Left err -> return $ throwErr $ Err $ show err
691+ Right EventNone -> return iter
692+ Right ev -> do
693+ igv <- runIter iter $ Chunk $ SC.fromList [ev]
694+ case igv of
695+ Done a _ -> return $ return a
696+ Cont iter' Nothing -> runParser fp iter'
697+ Cont _ (Just err) -> return $ throwErr err
698+
699+parserParseOne' :: Parser
700+ -> IO (Either YamlException Event)
701+parserParseOne' parser = allocaBytesR eventSize $ \er -> do
702+ res <- liftIO $ c_yaml_parser_parse parser er
703+ flip finally (c_yaml_event_delete er) $
704+ if res == 0
705+ then do
706+ problem <- makeString c_get_parser_error_problem parser
707+ context <- makeString c_get_parser_error_context parser
708+ offset <- fromIntegral `fmap`
709+ c_get_parser_error_offset parser
710+ return $ Left $ YamlParserException problem context offset
711+ else liftIO $ Right <$> getEvent er
runtests.hs+102−102
@@ -10,13 +10,12 @@ import "transformers" Control.Monad.Trans.Class
1010import "transformers" Control.Monad.Trans
1111#endif
1212
13−--import Test.Framework (testGroup, Test)
1413import Test.Framework.Providers.HUnit
15−--import Test.Framework.Providers.QuickCheck (testProperty)
1614import Test.HUnit hiding (Test, path)
17−--import Test.QuickCheck
1815
1916import Control.Exception
17+import Data.Iteratee hiding (filter, length, foldl')
18+import Data.List (foldl')
2019
2120main :: IO ()
2221main = defaultMain
@@ -28,137 +27,133 @@ main = defaultMain
2827 , testCase "largest string" caseLargestString
2928 , testCase "encode/decode" caseEncodeDecode
3029 , testCase "encode/decode file" caseEncodeDecodeFile
31− , testCase "interleaved encode/decode" caseInterleave
30+ --, testCase "interleaved encode/decode" caseInterleave
3231 , testCase "decode invalid document (without segfault)" caseDecodeInvalidDocument
3332 ]
3433
34+counter :: (Event -> Bool) -> Int -> IterateeG [] Event IO Int
35+counter pred' acc =
36+ IterateeG go
37+ where
38+ go (EOF x) = return $ Done acc $ EOF x
39+ go (Chunk c) =
40+ let acc' = length (filter pred' c) + acc
41+ in return $ Cont (counter pred' acc') Nothing
42+
43+caseHelper :: String
44+ -> (Event -> Bool)
45+ -> Int
46+ -> Assertion
47+caseHelper yamlString pred' expRes = do
48+ res <- decode (B8.pack yamlString) (counter pred' 0) >>= run
49+ res @?= expRes
50+
3551caseCountScalarsWithAnchor :: Assertion
36−caseCountScalarsWithAnchor = do
37− res <- decode yamlBS $ counter (0 :: Int)
38− res @?= (1 :: Int)
39− where
40− yamlString = "foo:\n - &anchor bin1\n - bin2\n - bin3"
41− yamlBS = B8.pack yamlString
42− counter acc = do
43− e <- parseEvent
44− case e of
45− EventScalar _ _ _ (Just _) -> counter (acc + 1)
46− EventNone -> return acc
47− _ -> counter acc
52+caseCountScalarsWithAnchor =
53+ caseHelper yamlString isScalarA 1
54+ where
55+ yamlString = "foo:\n - &anchor bin1\n - bin2\n - bin3"
56+ isScalarA (EventScalar _ _ _ (Just _)) = True
57+ isScalarA _ = False
4858
4959caseCountSequencesWithAnchor :: Assertion
50−caseCountSequencesWithAnchor = do
51− res <- decode yamlBS $ counter (0 :: Int)
52− res @?= (1 :: Int)
53− where
54− yamlString = "foo: &anchor\n - bin1\n - bin2\n - bin3"
55− yamlBS = B8.pack yamlString
56− counter acc = do
57− e <- parseEvent
58− case e of
59− EventSequenceStart (Just _) -> counter (acc + 1)
60− EventNone -> return acc
61− _ -> counter acc
60+caseCountSequencesWithAnchor =
61+ caseHelper yamlString isSequenceStartA 1
62+ where
63+ yamlString = "foo: &anchor\n - bin1\n - bin2\n - bin3"
64+ isSequenceStartA (EventSequenceStart (Just _)) = True
65+ isSequenceStartA _ = False
6266
6367caseCountMappingsWithAnchor :: Assertion
64−caseCountMappingsWithAnchor = do
65− res <- decode yamlBS $ counter (0 :: Int)
66− res @?= (1 :: Int)
67− where
68− yamlString = "foo: &anchor\n key1: bin1\n key2: bin2\n key3: bin3"
69− yamlBS = B8.pack yamlString
70− counter acc = do
71− e <- parseEvent
72− case e of
73− EventMappingStart (Just _) -> counter (acc + 1)
74− EventNone -> return acc
75− _ -> counter acc
68+caseCountMappingsWithAnchor =
69+ caseHelper yamlString isMappingA 1
70+ where
71+ yamlString = "foo: &anchor\n key1: bin1\n key2: bin2\n key3: bin3"
72+ isMappingA (EventMappingStart (Just _)) = True
73+ isMappingA _ = False
7674
7775caseCountAliases :: Assertion
78−caseCountAliases = do
79− res <- decode yamlBS $ counter (0 :: Int)
80− res @?= (1 :: Int)
81− where
82− yamlString = "foo: &anchor\n key1: bin1\n key2: bin2\n key3: bin3\nboo: *anchor"
83− yamlBS = B8.pack yamlString
84− counter acc = do
85− e <- parseEvent
86− case e of
87− EventAlias _ -> counter (acc + 1)
88− EventNone -> return acc
89− _ -> counter acc
76+caseCountAliases =
77+ caseHelper yamlString isAlias 1
78+ where
79+ yamlString = "foo: &anchor\n key1: bin1\n key2: bin2\n key3: bin3\nboo: *anchor"
80+ isAlias EventAlias{} = True
81+ isAlias _ = False
9082
9183caseCountScalars :: Assertion
9284caseCountScalars = do
93− res <- decode yamlBS $ counter accum
85+ res <- decode yamlBS (counter' accum) >>= run
9486 res @?= (7, 1, 2)
9587 where
9688 yamlString = "foo:\n baz: [bin1, bin2, bin3]\nbaz: bazval"
9789 yamlBS = B8.pack yamlString
98− counter (s, l, m) = do
99− e <- parseEvent
100− case e of
101− EventScalar {} -> counter (s + 1, l, m)
102− EventSequenceStart _ -> counter (s, l + 1, m)
103− EventMappingStart _ -> counter (s, l, m + 1)
104− EventNone -> return (s, l, m)
105− _ -> counter (s, l, m)
90+ counter' acc = IterateeG $ \s ->
91+ case s of
92+ EOF x -> return $ Done acc $ EOF x
93+ Chunk c ->
94+ let acc' = foldl' adder acc c
95+ in return $ Cont (counter' acc') Nothing
96+ adder (s, l, m) (EventScalar{}) = (s + 1, l, m)
97+ adder (s, l, m) (EventSequenceStart{}) = (s, l + 1, m)
98+ adder (s, l, m) (EventMappingStart{}) = (s, l, m + 1)
99+ adder a _ = a
106100 accum = (0, 0, 0) :: (Int, Int, Int)
107101
108102caseLargestString :: Assertion
109103caseLargestString = do
110− res <- decodeFile filePath $ dec accum
104+ res <- decodeFile filePath (dec accum) >>= run
111105 res @?= (length expected, expected)
112106 where
113107 expected = "this one is just a little bit bigger than the others"
114108 filePath = "test/largest-string.yaml"
115− dec (i, s) = do
116− e <- parseEvent
117− case e of
118− (EventScalar bs _ _ _) -> do
119− let s' = B8.unpack bs
120− i' = length s'
121− dec $ if i' > i then (i', s') else (i, s)
122− EventNone -> return (i, s)
123− _ -> dec (i, s)
109+ dec acc = IterateeG $ \s ->
110+ case s of
111+ EOF x -> return $ Done acc $ EOF x
112+ Chunk c ->
113+ let acc' = foldl' adder acc c
114+ in return $ Cont (dec acc') Nothing
115+ adder (i, s) (EventScalar bs _ _ _) =
116+ let s' = B8.unpack bs
117+ i' = length s'
118+ in if i' > i then (i', s') else (i, s)
119+ adder acc _ = acc
124120 accum = (0, "no strings found")
125121
122+newtype MyEvent = MyEvent Event deriving Show
123+instance Eq MyEvent where
124+ (MyEvent (EventScalar s t _ _)) == (MyEvent (EventScalar s' t' _ _)) =
125+ s == s' && t == t'
126+ MyEvent e1 == MyEvent e2 = e1 == e2
127+
126128caseEncodeDecode :: Assertion
127129caseEncodeDecode = do
128− eList <- decode yamlBS $ dec id
130+ eList <- decode yamlBS (dec []) >>= run
129131 bs <- encode $ mapM_ emitEvent eList
130− eList2 <- decode bs $ dec id
132+ eList2 <- decode bs (dec []) >>= run
131133 map MyEvent eList @=? map MyEvent eList2
132134 where
133135 yamlString = "foo: bar\nbaz:\n - bin1\n - bin2\n"
134136 yamlBS = B8.pack yamlString
135− dec front = do
136− e <- parseEvent
137− case e of
138− EventNone -> return $ front []
139− _ -> dec $ front . (:) e
137+ dec front = IterateeG $ \s ->
138+ case s of
139+ EOF x -> return $ Done front $ EOF x
140+ Chunk c -> return $ Cont (dec $ front ++ c) Nothing
140141
141142caseEncodeDecodeFile :: Assertion
142143caseEncodeDecodeFile = do
143− eList <- decodeFile filePath $ dec id
144+ eList <- decodeFile filePath (dec []) >>= run
144145 encodeFile tmpPath $ mapM_ emitEvent eList
145− eList2 <- decodeFile filePath $ dec id
146+ eList2 <- decodeFile filePath (dec []) >>= run
146147 map MyEvent eList @=? map MyEvent eList2
147148 where
148149 filePath = "test/largest-string.yaml"
149150 tmpPath = "tmp.yaml"
150− dec front = do
151− e <- parseEvent
152− case e of
153− EventNone -> return $ front []
154− _ -> dec $ front . (:) e
155−
156−newtype MyEvent = MyEvent Event deriving Show
157−instance Eq MyEvent where
158− (MyEvent (EventScalar s t _ _)) == (MyEvent (EventScalar s' t' _ _)) =
159− s == s' && t == t'
160− MyEvent e1 == MyEvent e2 = e1 == e2
151+ dec front = IterateeG $ \s ->
152+ case s of
153+ EOF x -> return $ Done front $ EOF x
154+ Chunk c -> return $ Cont (dec $ front ++ c) Nothing
161155
156+{-
162157caseInterleave :: Assertion
163158caseInterleave = do
164159 decodeFile filePath $ encodeFile tmpPath inside
@@ -176,17 +171,22 @@ caseInterleave = do
176171 case e of
177172 EventNone -> return ()
178173 _ -> emitEvent e >> inside
174+-}
179175
180176caseDecodeInvalidDocument :: Assertion
181177caseDecodeInvalidDocument = do
182− handle (\(YamlParserException {}) -> return ()) $ do
183− _ <- decode yamlBS $ dec id
184− assertFailure "expected parsing exception, but got no errors"
185− where
186− yamlString = " - foo\n - baz\nbuz"
187− yamlBS = B8.pack yamlString
188− dec front = do
189− e <- parseEvent
190− case e of
191− EventNone -> return $ front []
192− _ -> dec $ front . (:) e
178+ x <- decode yamlBS ignore
179+ y <- runIter x $ EOF Nothing
180+ case y of
181+ Cont _ _ -> return ()
182+ _ -> do
183+ putStrLn $ "bad return value: " ++ show y
184+ assertFailure "expected parsing exception, but got no errors"
185+ where
186+ yamlString = " - foo\n - baz\nbuz"
187+ yamlBS = B8.pack yamlString
188+ ignore :: IterateeG [] Event IO ()
189+ ignore = IterateeG $ \s ->
190+ case s of
191+ EOF x -> return $ Done () $ EOF x
192+ Chunk _c -> return $ Cont ignore Nothing
yaml.cabal+18−3
@@ -1,5 +1,5 @@
11name: yaml
2−version: 0.4.0
2+version: 0.5.0
33license: BSD3
44license-file: LICENSE
55author: Michael Snoyman <michael@snoyman.com>
@@ -15,15 +15,24 @@ extra-source-files: c/helper.h,
1515 c/yaml_private.h,
1616 c/yaml.h
1717
18+flag nolib
19+ description: Skip building the library
20+ default: False
1821flag buildtests
1922 description: Build the executable to run unit tests
2023 default: False
2124
2225library
26+ if flag(nolib)
27+ Buildable: False
28+ else
29+ Buildable: True
2330 build-depends: base >= 4 && < 5,
2431 transformers >= 0.1 && < 0.3,
2532 control-monad-failure >= 0.6.0 && < 0.7,
26− bytestring >= 0.9.1.4 && < 0.10
33+ bytestring >= 0.9.1.4 && < 0.10,
34+ iteratee >= 0.3.5 && < 0.4,
35+ MonadCatchIO-transformers >= 0.2.2 && < 0.3
2736 exposed-modules: Text.Libyaml
2837 ghc-options: -Wall -auto-all -caf-all
2938 c-sources: c/helper.c,
@@ -43,7 +52,13 @@ executable runtests
4352 cpp-options: -DTEST
4453 build-depends: test-framework,
4554 test-framework-hunit,
46− HUnit
55+ base >= 4 && < 5,
56+ HUnit,
57+ transformers >= 0.1 && < 0.3,
58+ control-monad-failure >= 0.6.0 && < 0.7,
59+ bytestring >= 0.9.1.4 && < 0.10,
60+ iteratee >= 0.3.5 && < 0.4,
61+ MonadCatchIO-transformers >= 0.2.2 && < 0.3
4762 else
4863 Buildable: False
4964 ghc-options: -Wall