on master help sign in
Merge pull request #181 from robbiemcmichael/reduce-encoder-duplication

Reduce duplication between encode and encodePretty

commit40a68af984d6e7368abd6e899cfbd7da400f189f authorMichael Snoyman <michael@snoyman.com> dateWed 30 Oct 2019, 05:53 (7 years ago) parent485df7b8868235
6 files changed+149−114
yaml/ChangeLog.md+7 yaml/src/Data/Yaml.hs+10−75 yaml/src/Data/Yaml/Builder.hs+2−16 yaml/src/Data/Yaml/Internal.hs+89−3 yaml/src/Data/Yaml/Pretty.hs+2−1 yaml/test/Data/YamlSpec.hs+39−19
yaml/ChangeLog.md+7
@@ -1,5 +1,12 @@
11# ChangeLog for yaml
22
3+## 0.11.2.0
4+
5+* Reduces some of the code duplication between the `encode` and `encodePretty` functions
6+* The output of `encodePretty` has been improved:
7+ - Multiline strings now use `Literal` style instead of `SingleQuoted`
8+ - Special keys are now quoted in maps [#179](https://github.com/snoyberg/yaml/issues/179)
9+
310## 0.11.1.2
411
512* Compiles with GHC 8.8.1 (`MonadFail` split)
yaml/src/Data/Yaml.hs+10−75
@@ -72,6 +72,7 @@ module Data.Yaml
7272 , isSpecialString
7373 , EncodeOptions
7474 , defaultEncodeOptions
75+ , defaultStringStyle
7576 , setStringStyle
7677 , setFormat
7778 , FormatOptions
@@ -94,18 +95,10 @@ import Data.Aeson
9495 , Object, Array
9596 , withObject, withText, withArray, withScientific, withBool
9697 )
97−import qualified Data.Scientific as S
98−import qualified Data.ByteString.Builder.Scientific
99−import Data.Aeson.Types (Pair, parseMaybe, parseEither, Parser)
98+import Data.Aeson.Types (parseMaybe, parseEither, Parser)
10099import Data.ByteString (ByteString)
101−import qualified Data.ByteString.Builder as BB
102−import qualified Data.ByteString.Lazy as BL
103100import Data.Conduit ((.|), runConduitRes)
104101import qualified Data.Conduit.List as CL
105−import qualified Data.HashMap.Strict as M
106−import qualified Data.HashSet as HashSet
107−import Data.Text.Encoding (encodeUtf8)
108−import qualified Data.Text as T
109102import qualified Data.Vector as V
110103import System.IO.Unsafe (unsafePerformIO)
111104import Data.Text (Text)
@@ -114,13 +107,6 @@ import Data.Yaml.Internal
114107import Text.Libyaml hiding (encode, decode, encodeFile, decodeFile, encodeWith, encodeFileWith)
115108import qualified Text.Libyaml as Y
116109
117−-- |
118−-- @since 0.10.2.0
119−data EncodeOptions = EncodeOptions
120− { encodeOptionsStringStyle :: Text -> ( Tag, Style )
121− , encodeOptionsFormat :: FormatOptions
122− }
123−
124110-- | Set the string style in the encoded YAML. This is a function that decides
125111-- for each string the type of YAML string to output.
126112--
@@ -145,25 +131,18 @@ setStringStyle s opts = opts { encodeOptionsStringStyle = s }
145131setFormat :: FormatOptions -> EncodeOptions -> EncodeOptions
146132setFormat f opts = opts { encodeOptionsFormat = f }
147133
148−-- | Determine whether a string must be quoted in YAML and can't appear as plain text.
149−-- Useful if you want to use 'setStringStyle'.
150−--
134+-- |
151135-- @since 0.10.2.0
152−isSpecialString :: Text -> Bool
153−isSpecialString s = s `HashSet.member` specialStrings || isNumeric s
136+data EncodeOptions = EncodeOptions
137+ { encodeOptionsStringStyle :: Text -> ( Tag, Style )
138+ , encodeOptionsFormat :: FormatOptions
139+ }
154140
155141-- |
156142-- @since 0.10.2.0
157143defaultEncodeOptions :: EncodeOptions
158144defaultEncodeOptions = EncodeOptions
159− { encodeOptionsStringStyle = \s ->
160− -- Empty strings need special handling to ensure they get quoted. This avoids:
161− -- https://github.com/snoyberg/yaml/issues/24
162− case () of
163− ()
164− | "\n" `T.isInfixOf` s -> ( NoTag, Literal )
165− | isSpecialString s -> ( NoTag, SingleQuoted )
166− | otherwise -> ( StrTag, PlainNoTag )
145+ { encodeOptionsStringStyle = defaultStringStyle
167146 , encodeOptionsFormat = defaultFormatOptions
168147 }
169148
@@ -176,7 +155,7 @@ encode = encodeWith defaultEncodeOptions
176155-- @since 0.10.2.0
177156encodeWith :: ToJSON a => EncodeOptions -> a -> ByteString
178157encodeWith opts obj = unsafePerformIO $ runConduitRes
179− $ CL.sourceList (objToEvents opts $ toJSON obj)
158+ $ CL.sourceList (objToStream (encodeOptionsStringStyle opts) $ toJSON obj)
180159 .| Y.encodeWith (encodeOptionsFormat opts)
181160
182161-- | Encode a value into its YAML representation and save to the given file.
@@ -188,53 +167,9 @@ encodeFile = encodeFileWith defaultEncodeOptions
188167-- @since 0.10.2.0
189168encodeFileWith :: ToJSON a => EncodeOptions -> FilePath -> a -> IO ()
190169encodeFileWith opts fp obj = runConduitRes
191− $ CL.sourceList (objToEvents opts $ toJSON obj)
170+ $ CL.sourceList (objToStream (encodeOptionsStringStyle opts) $ toJSON obj)
192171 .| Y.encodeFileWith (encodeOptionsFormat opts) fp
193172
194−objToEvents :: EncodeOptions -> Value -> [Y.Event]
195−objToEvents opts o = (:) EventStreamStart
196− . (:) EventDocumentStart
197− $ objToEvents' o
198− [ EventDocumentEnd
199− , EventStreamEnd
200− ]
201− where
202− objToEvents' :: Value -> [Y.Event] -> [Y.Event]
203− --objToEvents' (Scalar s) rest = scalarToEvent s : rest
204− objToEvents' (Array list) rest =
205− EventSequenceStart NoTag AnySequence Nothing
206− : foldr objToEvents' (EventSequenceEnd : rest) (V.toList list)
207− objToEvents' (Object pairs) rest =
208− EventMappingStart NoTag AnyMapping Nothing
209− : foldr pairToEvents (EventMappingEnd : rest) (M.toList pairs)
210−
211− objToEvents' (String "") rest = EventScalar "" NoTag SingleQuoted Nothing : rest
212−
213− objToEvents' (String s) rest = EventScalar (encodeUtf8 s) tag style Nothing : rest
214− where
215− ( tag, style ) = encodeOptionsStringStyle opts s
216− objToEvents' Null rest = EventScalar "null" NullTag PlainNoTag Nothing : rest
217− objToEvents' (Bool True) rest = EventScalar "true" BoolTag PlainNoTag Nothing : rest
218− objToEvents' (Bool False) rest = EventScalar "false" BoolTag PlainNoTag Nothing : rest
219−
220− objToEvents' (Number s) rest =
221− let builder
222− -- Special case the 0 exponent to remove the trailing .0
223− | S.base10Exponent s == 0 = BB.integerDec $ S.coefficient s
224− | otherwise = Data.ByteString.Builder.Scientific.scientificBuilder s
225− lbs = BB.toLazyByteString builder
226− bs = BL.toStrict lbs
227− in EventScalar bs IntTag PlainNoTag Nothing : rest
228−
229− pairToEvents :: Pair -> [Y.Event] -> [Y.Event]
230− pairToEvents (k, v) = objToEvents' (String k) . objToEvents' v
231−
232−{- FIXME
233−scalarToEvent :: YamlScalar -> Event
234−scalarToEvent (YamlScalar v t s) = EventScalar v t s Nothing
235−-}
236−
237−
238173decode :: FromJSON a
239174 => ByteString
240175 -> Maybe a
yaml/src/Data/Yaml/Builder.hs+2−16
@@ -47,10 +47,8 @@ import Data.Aeson.Types (Value(..))
4747import Data.ByteString (ByteString)
4848import qualified Data.ByteString.Char8 as S8
4949import Data.Conduit
50−import qualified Data.HashSet as HashSet
5150import Data.Scientific (Scientific)
5251import Data.Text (Text, unpack)
53−import Data.Text.Encoding (encodeUtf8)
5452import qualified Data.Text.Encoding as TE
5553import qualified Data.Text.Lazy as TL
5654import Data.Text.Lazy.Builder (toLazyText)
@@ -83,9 +81,7 @@ maybeNamedMapping :: Maybe Text -> [(Text, YamlBuilder)] -> YamlBuilder
8381maybeNamedMapping anchor pairs = YamlBuilder $ \rest ->
8482 EventMappingStart NoTag AnyMapping (unpack <$> anchor) : foldr addPair (EventMappingEnd : rest) pairs
8583 where
86− addPair (key, YamlBuilder value) after
87− = EventScalar (encodeUtf8 key) StrTag PlainNoTag Nothing
88− : value after
84+ addPair (key, YamlBuilder value) after = unYamlBuilder (string key) $ value after
8985
9086mapping :: [(Text, YamlBuilder)] -> YamlBuilder
9187mapping = maybeNamedMapping Nothing
@@ -114,17 +110,7 @@ namedArray name = maybeNamedArray $ Just name
114110-- |
115111-- @since 0.11.0
116112maybeNamedString :: Maybe Text -> Text -> YamlBuilder
117−-- Empty strings need special handling to ensure they get quoted. This avoids:
118−-- https://github.com/snoyberg/yaml/issues/24
119−maybeNamedString anchor "" = YamlBuilder (EventScalar "" NoTag SingleQuoted (unpack <$> anchor) :)
120−maybeNamedString anchor s =
121− YamlBuilder (event :)
122− where
123− event
124− -- Make sure that special strings are encoded as strings properly.
125− -- See: https://github.com/snoyberg/yaml/issues/31
126− | s `HashSet.member` specialStrings || isNumeric s = EventScalar (encodeUtf8 s) NoTag SingleQuoted $ unpack <$> anchor
127− | otherwise = EventScalar (encodeUtf8 s) StrTag PlainNoTag $ unpack <$> anchor
113+maybeNamedString anchor s = YamlBuilder (stringScalar defaultStringStyle anchor s :)
128114
129115string :: Text -> YamlBuilder
130116string = maybeNamedString Nothing
yaml/src/Data/Yaml/Internal.hs+89−3
@@ -11,9 +11,14 @@ module Data.Yaml.Internal
1111 , parse
1212 , decodeHelper
1313 , decodeHelper_
14+ , textToScientific
15+ , stringScalar
16+ , defaultStringStyle
17+ , isSpecialString
1418 , specialStrings
1519 , isNumeric
16− , textToScientific
20+ , objToStream
21+ , objToEvents
1722 ) where
1823
1924#if !MIN_VERSION_base(4,8,0)
@@ -31,6 +36,9 @@ import Data.Aeson.Types hiding (parse)
3136import qualified Data.Attoparsec.Text as Atto
3237import Data.Bits (shiftL, (.|.))
3338import Data.ByteString (ByteString)
39+import qualified Data.ByteString.Builder as BB
40+import qualified Data.ByteString.Lazy as BL
41+import Data.ByteString.Builder.Scientific (scientificBuilder)
3442import Data.Char (toUpper, ord)
3543import Data.List
3644import Data.Conduit ((.|), ConduitM, runConduit)
@@ -41,10 +49,10 @@ import Data.Map (Map)
4149import qualified Data.Map as Map
4250import Data.Set (Set)
4351import qualified Data.Set as Set
44−import Data.Scientific (Scientific)
52+import Data.Scientific (Scientific, base10Exponent, coefficient)
4553import Data.Text (Text, pack)
4654import qualified Data.Text as T
47−import Data.Text.Encoding (decodeUtf8With)
55+import Data.Text.Encoding (decodeUtf8With, encodeUtf8)
4856import Data.Text.Encoding.Error (lenientDecode)
4957import Data.Typeable
5058import qualified Data.Vector as V
@@ -301,10 +309,88 @@ decodeHelper_ src = do
301309 Right
302310 ((,) (parseStateWarnings st) <$> parseEither parseJSON y)
303311
312+type StringStyle = Text -> ( Tag, Style )
313+
314+-- | Encodes a string with the supplied style. This function handles the empty
315+-- string case properly to avoid https://github.com/snoyberg/yaml/issues/24
316+--
317+-- @since 0.11.2.0
318+stringScalar :: StringStyle -> Maybe Text -> Text -> Event
319+stringScalar _ anchor "" = EventScalar "" NoTag SingleQuoted (T.unpack <$> anchor)
320+stringScalar stringStyle anchor s = EventScalar (encodeUtf8 s) tag style (T.unpack <$> anchor)
321+ where
322+ ( tag, style ) = stringStyle s
323+
324+-- |
325+-- @since 0.11.2.0
326+defaultStringStyle :: StringStyle
327+defaultStringStyle = \s ->
328+ case () of
329+ ()
330+ | "\n" `T.isInfixOf` s -> ( NoTag, Literal )
331+ | isSpecialString s -> ( NoTag, SingleQuoted )
332+ | otherwise -> ( StrTag, PlainNoTag )
333+
334+-- | Determine whether a string must be quoted in YAML and can't appear as plain text.
335+-- Useful if you want to use 'setStringStyle'.
336+--
337+-- @since 0.10.2.0
338+isSpecialString :: Text -> Bool
339+isSpecialString s = s `HashSet.member` specialStrings || isNumeric s
340+
304341-- | Strings which must be escaped so as not to be treated as non-string scalars.
342+--
343+-- @since 0.8.32
305344specialStrings :: HashSet.HashSet Text
306345specialStrings = HashSet.fromList $ T.words
307346 "y Y yes Yes YES n N no No NO true True TRUE false False FALSE on On ON off Off OFF null Null NULL ~ *"
308347
348+-- |
349+-- @since 0.8.32
309350isNumeric :: Text -> Bool
310351isNumeric = either (const False) (const True) . textToScientific
352+
353+-- | Encode a value as a YAML document stream.
354+--
355+-- @since 0.11.2.0
356+objToStream :: ToJSON a => StringStyle -> a -> [Y.Event]
357+objToStream stringStyle o =
358+ (:) EventStreamStart
359+ . (:) EventDocumentStart
360+ $ objToEvents stringStyle o
361+ [ EventDocumentEnd
362+ , EventStreamEnd
363+ ]
364+
365+-- | Encode a value as a list of 'Event's.
366+--
367+-- @since 0.11.2.0
368+objToEvents :: ToJSON a => StringStyle -> a -> [Y.Event] -> [Y.Event]
369+objToEvents stringStyle = objToEvents' . toJSON
370+ where
371+ objToEvents' (Array list) rest =
372+ EventSequenceStart NoTag AnySequence Nothing
373+ : foldr objToEvents' (EventSequenceEnd : rest) (V.toList list)
374+
375+ objToEvents' (Object o) rest =
376+ EventMappingStart NoTag AnyMapping Nothing
377+ : foldr pairToEvents (EventMappingEnd : rest) (M.toList o)
378+ where
379+ pairToEvents :: Pair -> [Y.Event] -> [Y.Event]
380+ pairToEvents (k, v) = objToEvents' (String k) . objToEvents' v
381+
382+ objToEvents' (String s) rest = stringScalar stringStyle Nothing s : rest
383+
384+ objToEvents' Null rest = EventScalar "null" NullTag PlainNoTag Nothing : rest
385+
386+ objToEvents' (Bool True) rest = EventScalar "true" BoolTag PlainNoTag Nothing : rest
387+ objToEvents' (Bool False) rest = EventScalar "false" BoolTag PlainNoTag Nothing : rest
388+
389+ objToEvents' (Number s) rest =
390+ let builder
391+ -- Special case the 0 exponent to remove the trailing .0
392+ | base10Exponent s == 0 = BB.integerDec $ coefficient s
393+ | otherwise = scientificBuilder s
394+ lbs = BB.toLazyByteString builder
395+ bs = BL.toStrict lbs
396+ in EventScalar bs IntTag PlainNoTag Nothing : rest
yaml/src/Data/Yaml/Pretty.hs+2−1
@@ -10,6 +10,7 @@ module Data.Yaml.Pretty
1010 , getConfDropNull
1111 , setConfDropNull
1212 , defConfig
13+ , pretty
1314 ) where
1415
1516import Prelude hiding (null)
@@ -34,7 +35,7 @@ import Data.Yaml.Builder
3435-- @since 0.8.13
3536data Config = Config
3637 { confCompare :: Text -> Text -> Ordering -- ^ Function used to sort keys in objects
37− , confDropNull :: Bool
38+ , confDropNull :: Bool -- ^ Drop null values from objects
3839 }
3940
4041-- | The default configuration: do not sort objects or drop keys
yaml/test/Data/YamlSpec.hs+39−19
@@ -27,6 +27,7 @@ import System.Directory (createDirectory, createDirectoryIfMissing)
2727import Test.Mockery.Directory
2828
2929import qualified Data.Yaml as D
30+import qualified Data.Yaml.Internal as Internal
3031import qualified Data.Yaml.Pretty as Pretty
3132import Data.Yaml (object, array, (.=))
3233import Data.Maybe
@@ -53,6 +54,15 @@ data TestJSON = TestJSON
5354
5455deriveJSON defaultOptions ''TestJSON
5556
57+testJSON :: TestJSON
58+testJSON = TestJSON
59+ { string = "str"
60+ , number = 2
61+ , anArray = V.fromList ["a", "b"]
62+ , hash = HM.fromList [("key1", "value1"), ("key2", "value2")]
63+ , extrastring = "1234-foo"
64+ }
65+
5666shouldDecode :: (Show a, D.FromJSON a, Eq a) => B8.ByteString -> a -> IO ()
5767shouldDecode bs expected = do
5868 actual <- D.decodeThrow bs
@@ -86,6 +96,7 @@ spec = do
8696 it "count scalars" caseCountScalars
8797 it "largest string" caseLargestString
8898 it "encode/decode" caseEncodeDecode
99+ it "encode/decode events" caseEncodeDecodeEvents
89100 it "encode/decode file" caseEncodeDecodeFile
90101 it "interleaved encode/decode" caseInterleave
91102 it "decode invalid document (without segfault)" caseDecodeInvalidDocument
@@ -159,13 +170,7 @@ spec = do
159170 describe "decodeFileEither" $ do
160171 it "loads YAML through JSON into Haskell data" $ do
161172 tj <- either (error . show) id `fmap` D.decodeFileEither "test/json.yaml"
162− tj `shouldBe` TestJSON
163− { string = "str"
164− , number = 2
165− , anArray = V.fromList ["a", "b"]
166− , hash = HM.fromList [("key1", "value1"), ("key2", "value2")]
167− , extrastring = "1234-foo"
168− }
173+ tj `shouldBe` testJSON
169174
170175 context "when file does not exist" $ do
171176 it "returns Left" $ do
@@ -188,18 +193,9 @@ spec = do
188193
189194 it "truncates files" caseTruncatesFiles
190195
191− it "quoting keys #137" $ do
192− let keys = T.words "true false NO YES 1.2 1e5 null"
193− bs = D.encode $ M.fromList $ map (, ()) keys
194− text = decodeUtf8 bs
195− forM_ keys $ \key -> do
196− let quoted = T.concat ["'", key, "'"]
197− unless (quoted `T.isInfixOf` text) $ error $ concat
198− [ "Could not find quoted key: "
199− , T.unpack quoted
200− , "\n\n"
201− , T.unpack text
202− ] :: IO ()
196+ it "encode quotes special keys #137" $ caseSpecialKeys D.encode
197+
198+ it "encodePretty quotes special keys #179" $ caseSpecialKeys (Pretty.encodePretty Pretty.defConfig)
203199
204200 describe "non-decimal numbers #135" $ do
205201 let go str val = it str $ encodeUtf8 (T.pack str) `shouldDecode` val
@@ -479,6 +475,17 @@ caseEncodeDecode = do
479475 yamlString = "foo: bar\nbaz:\n - bin1\n - bin2\n"
480476 yamlBS = B8.pack yamlString
481477
478+caseEncodeDecodeEvents :: Assertion
479+caseEncodeDecodeEvents = do
480+ let events = Internal.objToEvents D.defaultStringStyle testJSON []
481+ result <- Internal.decodeHelper_ . CL.sourceList $ toStream events
482+ let (_, value) = either (error . show) id result
483+ value @?= testJSON
484+ where
485+ toStream es =
486+ [Y.EventStreamStart, Y.EventDocumentStart] ++ es ++
487+ [Y.EventDocumentEnd, Y.EventStreamEnd]
488+
482489caseEncodeDecodeFile :: Assertion
483490caseEncodeDecodeFile = withFile "" $ \tmpPath -> do
484491 eList <- runConduitRes $ Y.decodeFile filePath .| CL.consume
@@ -737,6 +744,19 @@ caseTruncatesFiles = withSystemTempFile "truncate.yaml" $ \fp h -> do
737744 res <- D.decodeFileEither fp
738745 either (Left . show) Right res `shouldBe` Right val
739746
747+caseSpecialKeys :: (HashMap Text () -> B8.ByteString) -> Assertion
748+caseSpecialKeys encoder = do
749+ let keys = T.words "true false NO YES 1.2 1e5 null"
750+ bs = encoder $ M.fromList $ map (, ()) keys
751+ text = decodeUtf8 bs
752+ forM_ keys $ \key -> do
753+ let quoted = T.concat ["'", key, "'"]
754+ unless (quoted `T.isInfixOf` text) $ error $ concat
755+ [ "Could not find quoted key: "
756+ , T.unpack quoted
757+ , "\n\n"
758+ , T.unpack text
759+ ] :: IO ()
740760
741761taggedSequence :: [Y.Event]
742762taggedSequence =