on master help sign in
First stab at payload type annotation
commite700f8a34a1f9ea2552fd19a3f0c98e0bc3078e4 authorMichael Xavier <michael@michaelxavier.net> dateTue 2 Jun 2015, 16:11 (11 years ago) parentab81b5f
6 files changed+285−21
katip-elasticsearch/bench/Main.hs+29−3 katip-elasticsearch/katip-elasticsearch.cabal+11−2 katip-elasticsearch/src/Katip/Scribes/ElasticSearch.hs+39−9 katip-elasticsearch/src/Katip/Scribes/ElasticSearch/Annotations.hs+111 katip-elasticsearch/test/Main.hs+93−6 katip/src/Katip/Core.hs+2−1
katip-elasticsearch/bench/Main.hs+29−3
@@ -10,27 +10,53 @@ module Main
1010import Control.DeepSeq
1111import Control.Monad
1212import Criterion.Main
13+import Data.Aeson
14+import qualified Data.HashMap.Strict as HM
1315import Data.RNG
14−import qualified Data.Text as T
16+import qualified Data.Text as T
1517import Database.Bloodhound.Types
1618import Numeric
1719-------------------------------------------------------------------------------
1820import Katip.Scribes.ElasticSearch
21+import Katip.Scribes.ElasticSearch.Annotations
1922-------------------------------------------------------------------------------
2023
2124main :: IO ()
2225main = do
2326 rng <- mkRNG
24− defaultMain [mkDocIdBenchmark rng]
27+ defaultMain
28+ [
29+ mkDocIdBenchmark rng
30+ , deannotateValueBenchmark
31+ ]
2532
2633-------------------------------------------------------------------------------
2734mkDocIdBenchmark :: RNG -> Benchmark
2835mkDocIdBenchmark rng = bgroup "mkDocId"
29− [ bench "mkDocId (randomIO)" $ nfIO mkDocId
36+ [
37+ bench "mkDocId (randomIO)" $ nfIO mkDocId
3038 , bench "mkDocId' (shared )" $ nfIO $ mkDocId' rng
3139 ]
3240
3341
42+-------------------------------------------------------------------------------
43+deannotateValueBenchmark :: Benchmark
44+deannotateValueBenchmark = bgroup "deannotateValue"
45+ [
46+ bench "deannotateValue" $ nf deannotateValue annotatedValue
47+ ]
48+
49+
50+-------------------------------------------------------------------------------
51+annotatedValue :: Value
52+annotatedValue = Object $ HM.fromList [ ("a::string", String "whatever")
53+ , ("b::double", Number 4.5)
54+ , ("c::long", Number 4)
55+ , ("d::boolean", Bool True)
56+ , ("e::null", Null)
57+ ]
58+
59+-------------------------------------------------------------------------------
3460mkDocId' :: RNG -> IO DocId
3561mkDocId' rng = do
3662 is <- withRNG rng $ \gen -> replicateM len $ mk gen
katip-elasticsearch/katip-elasticsearch.cabal+11−2
@@ -16,6 +16,7 @@ flag lib-Werror
1616library
1717 exposed-modules:
1818 Katip.Scribes.ElasticSearch
19+ Katip.Scribes.ElasticSearch.Annotations
1920 build-depends:
2021 base >=4.6 && <4.9
2122 , katip
@@ -31,6 +32,8 @@ library
3132 , text
3233 , retry >= 0.6 && < 0.8
3334 , exceptions
35+ , scientific
36+ , unordered-containers
3437 hs-source-dirs: src
3538 default-language: Haskell2010
3639 hs-source-dirs: src
@@ -53,6 +56,8 @@ benchmark bench
5356 , rng-utils
5457 , uuid
5558 , text
59+ , unordered-containers
60+ , aeson
5661
5762test-suite test
5863 type: exitcode-stdio-1.0
@@ -65,8 +70,10 @@ test-suite test
6570 , http-client
6671 , bloodhound
6772 , base
68− , tasty
73+ , tasty >= 0.10.1.2 && < 0.11
6974 , tasty-hunit
75+ , tasty-quickcheck
76+ , quickcheck-instances
7077 , containers
7178 , transformers
7279 , lens
@@ -74,4 +81,6 @@ test-suite test
7481 , text
7582 , http-types
7683 , aeson
77−
84+ , vector
85+ , unordered-containers
86+ , scientific
katip-elasticsearch/src/Katip/Scribes/ElasticSearch.hs+39−9
@@ -15,13 +15,16 @@ module Katip.Scribes.ElasticSearch
1515 , essRetryPolicy
1616 , essQueueSize
1717 , essPoolSize
18+ , essAnnotateTypes
1819 , defaultEsScribeCfg
1920 -- * Utilities
2021 , mkDocId
22+ , module Katip.Scribes.ElasticSearch.Annotations
2123 ) where
2224
2325
2426-------------------------------------------------------------------------------
27+import Control.Applicative
2528import Control.Concurrent
2629import Control.Concurrent.Async
2730import Control.Concurrent.STM.TBMQueue
@@ -30,12 +33,13 @@ import Control.Exception.Enclosed
3033import Control.Monad
3134import Control.Monad.Catch
3235import Control.Monad.STM
33−import Control.Retry (RetryPolicy,
34− exponentialBackoff,
35− limitRetries, recovering)
36+import Control.Retry (RetryPolicy,
37+ exponentialBackoff,
38+ limitRetries,
39+ recovering)
3640import Data.Aeson
37−import Data.Monoid ((<>))
38−import qualified Data.Text.Encoding as T
41+import Data.Monoid ((<>))
42+import qualified Data.Text.Encoding as T
3943import Data.Typeable
4044import Data.UUID
4145import Database.Bloodhound
@@ -43,6 +47,7 @@ import Network.HTTP.Client
4347import System.Random
4448-------------------------------------------------------------------------------
4549import Katip.Core
50+import Katip.Scribes.ElasticSearch.Annotations
4651-------------------------------------------------------------------------------
4752
4853
@@ -55,6 +60,19 @@ data EsScribeCfg = EsScribeCfg {
5560 -- ^ Maximum size of the bounded log queue
5661 , essPoolSize :: EsPoolSize
5762 -- ^ Worker pool size limit for sending data to the
63+ , essAnnotateTypes :: Bool
64+ -- ^ Different payload items coexist in the "data" attribute in
65+ -- ES. It is possible for different payloads to have different
66+ -- types for the same key, e.g. an "id" key that is sometimes a
67+ -- number and sometimes a string. If you're having ES do dynamic
68+ -- mapping, the first log item will set the type and any that
69+ -- don't conform will be *discarded*. If you set this to True,
70+ -- keys will recursively be appended with their ES core
71+ -- type. e.g. "id" would become "id::long" and "id::string"
72+ -- automatically, so they won't conflict. When this library
73+ -- exposes a querying API, we will try to make deserialization and
74+ -- querying transparently remove the type annotations if this is
75+ -- enabled.
5876 } deriving (Typeable)
5977
6078
@@ -67,15 +85,23 @@ data EsScribeCfg = EsScribeCfg {
6785-- * Queue size of 1000
6886--
6987-- * Pool size of 2
88+--
89+-- * Annotate types set to False
7090defaultEsScribeCfg :: EsScribeCfg
7191defaultEsScribeCfg = EsScribeCfg {
7292 essManagerSettings = defaultManagerSettings
73− , essRetryPolicy = exponentialBackoff 25 <> limitRetries 5
74− , essQueueSize = EsQueueSize 1000
75− , essPoolSize = EsPoolSize 2
93+ , essRetryPolicy = exponentialBackoff 25 <> limitRetries 5
94+ , essQueueSize = EsQueueSize 1000
95+ , essPoolSize = EsPoolSize 2
96+ , essAnnotateTypes = False
7697 }
7798
7899
100+-------------------------------------------------------------------------------
101+
102+
103+
104+
79105-------------------------------------------------------------------------------
80106mkEsScribe
81107 :: EsScribeCfg
@@ -106,9 +132,13 @@ mkEsScribe cfg@EsScribeCfg {..} server ix mapping sev verb = do
106132
107133 let scribe = Scribe $ \ i ->
108134 when (_itemSeverity i >= sev) $
109− void $ atomically $ tryWriteTBMQueue q (itemJson verb i)
135+ void $ atomically $ tryWriteTBMQueue q (itemJson' verb i)
110136 let finalizer = putMVar endSig () >> takeMVar endSig
111137 return (scribe, finalizer)
138+ where
139+ itemJson' v i
140+ | essAnnotateTypes = itemJson v (TypeAnnotated <$> i)
141+ | otherwise = itemJson v i
112142
113143
114144-------------------------------------------------------------------------------
katip-elasticsearch/src/Katip/Scribes/ElasticSearch/Annotations.hsnew file+111
@@ -0,0 +1,111 @@
1+{-# LANGUAGE OverloadedStrings #-}
2+module Katip.Scribes.ElasticSearch.Annotations
3+ ( TypeAnnotated(..)
4+ -- * Exported for benchmarking
5+ , deannotateValue
6+ ) where
7+
8+
9+-------------------------------------------------------------------------------
10+import Control.Applicative
11+import Data.Aeson
12+import qualified Data.Foldable as FT
13+import qualified Data.HashMap.Strict as HM
14+import Data.Maybe
15+import Data.Monoid
16+import Data.Scientific (isFloating)
17+import Data.Text (Text)
18+import qualified Data.Text as T
19+-------------------------------------------------------------------------------
20+import Katip
21+-------------------------------------------------------------------------------
22+
23+
24+-- | Represents a value that can be converted to and from JSON that
25+-- will type annotate object keys when serializing and strip them out when deserializating
26+newtype TypeAnnotated a = TypeAnnotated {
27+ typeAnnotatedValue :: a
28+ }
29+
30+instance ToJSON a => ToJSON (TypeAnnotated a) where
31+ toJSON = annotateValue . toJSON . typeAnnotatedValue
32+
33+
34+instance ToObject a => ToObject (TypeAnnotated a)
35+
36+
37+instance FromJSON a => FromJSON (TypeAnnotated a) where
38+ parseJSON v = TypeAnnotated <$> parseJSON (deannotateValue v)
39+
40+
41+instance LogItem a => LogItem (TypeAnnotated a) where
42+ payloadKeys v (TypeAnnotated x) = case payloadKeys v x of
43+ AllKeys -> AllKeys
44+ -- Take the key selection, overlap it with the actual keys
45+ -- produced and annotate them
46+ SomeKeys ks -> let o = toObject x
47+ oInFocus = HM.fromList $ zip ks (repeat Null)
48+ final = annotateKeys $ HM.intersection o oInFocus
49+ in SomeKeys $ HM.keys final
50+
51+-------------------------------------------------------------------------------
52+-- Conversion Functions
53+-------------------------------------------------------------------------------
54+
55+
56+annotateValue :: Value -> Value
57+annotateValue (Object o) = Object $ annotateKeys o
58+annotateValue (Array a) = Array (annotateValue <$> a)
59+annotateValue x = x
60+
61+
62+annotateKeys :: Object -> Object
63+annotateKeys = HM.fromList . map go . HM.toList
64+ where
65+ go (k, Object o) = (k, Object $ annotateKeys o)
66+ go (k, Array a) = (k, Array (annotateValue <$> a))
67+ go (k, s@(String _)) = (k <> stringAnn, s)
68+ go (k, n@(Number sci)) = if isFloating sci
69+ then (k <> doubleAnn, n)
70+ else (k <> longAnn, n)
71+ go (k, b@(Bool _)) = (k <> booleanAnn, b)
72+ go (k, Null) = (k <> nullAnn, Null)
73+
74+
75+deannotateValue :: Value -> Value
76+deannotateValue (Object o) = Object $ deannotateKeys o
77+deannotateValue (Array a) = Array (deannotateValue <$> a)
78+deannotateValue x = x
79+
80+
81+deannotateKeys :: Object -> Object
82+deannotateKeys = HM.fromList . map go . HM.toList
83+ where
84+ go (k, Object o) = (k, Object $ deannotateKeys o)
85+ go (k, Array a) = (k, Array (deannotateValue <$> a))
86+ go (k, v) = (fromMaybe k k', v)
87+ where
88+ k' = FT.asum (stripSuffix <$> suffixes)
89+ suffixes = [stringAnn, doubleAnn, longAnn, booleanAnn, nullAnn]
90+ stripSuffix suffix = T.stripSuffix suffix k
91+
92+
93+-------------------------------------------------------------------------------
94+-- Annotation Constants
95+-------------------------------------------------------------------------------
96+
97+
98+stringAnn :: Text
99+stringAnn = "::string"
100+
101+doubleAnn :: Text
102+doubleAnn = "::double"
103+
104+longAnn :: Text
105+longAnn = "::long"
106+
107+booleanAnn :: Text
108+booleanAnn = "::boolean"
109+
110+nullAnn :: Text
111+nullAnn = "::null"
katip-elasticsearch/test/Main.hs+93−6
@@ -1,24 +1,32 @@
1−{-# LANGUAGE OverloadedStrings #-}
2−{-# LANGUAGE RecordWildCards #-}
3−{-# LANGUAGE TemplateHaskell #-}
1+{-# LANGUAGE OverloadedStrings #-}
2+{-# LANGUAGE RecordWildCards #-}
3+{-# LANGUAGE ScopedTypeVariables #-}
4+{-# LANGUAGE TemplateHaskell #-}
45module Main
56 ( main
67 ) where
78
89
910-------------------------------------------------------------------------------
10−import Control.Lens hiding (mapping)
11+import Control.Applicative
12+import Control.Lens hiding (mapping, (.=))
1113import Control.Monad
1214import Control.Monad.IO.Class
1315import Data.Aeson
1416import Data.Aeson.Lens
17+import Data.Aeson.Types
18+import qualified Data.HashMap.Strict as HM
1519import qualified Data.Map as M
1620import Data.Monoid
21+import Data.Scientific
22+import qualified Data.Vector as V
1723import Database.Bloodhound hiding (key)
1824import Network.HTTP.Client
1925import Network.HTTP.Types.Status
26+import Test.QuickCheck.Instances ()
2027import Test.Tasty
2128import Test.Tasty.HUnit
29+import Test.Tasty.QuickCheck
2230-------------------------------------------------------------------------------
2331import Katip
2432import Katip.Scribes.ElasticSearch
@@ -29,6 +37,7 @@ main :: IO ()
2937main = defaultMain $ testGroup "katip-elasticsearch"
3038 [
3139 withResource setupSearch teardownSearch esTests
40+ , typeAnnotatedTests
3241 ]
3342
3443
@@ -36,7 +45,7 @@ main = defaultMain $ testGroup "katip-elasticsearch"
3645setupSearch :: IO (Scribe, IO ())
3746setupSearch = do
3847 bh recreateESSchema
39− mkEsScribe defaultEsScribeCfg svr ixn mn DebugS V3
48+ mkEsScribe defaultEsScribeCfg { essAnnotateTypes = True } svr ixn mn DebugS V3
4049
4150
4251-------------------------------------------------------------------------------
@@ -51,13 +60,73 @@ esTests :: IO (Scribe, IO ()) -> TestTree
5160esTests setup = testGroup "elasticsearch scribe"
5261 [
5362 testCase "it flushes to elasticsearch" $ withTestLogging setup $ \done -> do
54− $(logT) () mempty InfoS "A test message"
63+ $(logT) (ExampleCtx True) mempty InfoS "A test message"
5564 liftIO $ do
5665 done
5766 logs <- getLogs
5867 length logs @?= 1
5968 let l = head logs
6069 l ^? key "_source" . key "msg" . _String @?= Just "A test message"
70+ l ^? key "_source" . key "data" . key "whatever::boolean" . _Bool @?= Just True
71+ ]
72+
73+
74+-------------------------------------------------------------------------------
75+data ExampleCtx = ExampleCtx {
76+ ecBool :: Bool
77+ }
78+
79+instance ToJSON ExampleCtx where
80+ toJSON c = object ["whatever" .= ecBool c]
81+
82+
83+instance ToObject ExampleCtx
84+
85+
86+instance LogItem ExampleCtx where
87+ payloadKeys _ _ = AllKeys
88+
89+-------------------------------------------------------------------------------
90+typeAnnotatedTests :: TestTree
91+typeAnnotatedTests = testGroup "TypeAnnotated"
92+ [
93+ testCase "annotates values on toJSON" $ do
94+ toJSON (TypeAnnotated exampleValue) @?= annotatedExampleValue
95+ , testCase "deannotates on parseJSON" $ do
96+ parseEither parseJSON (toJSON exampleValue) @?= Right exampleValue
97+ , testProperty "roundtrips the same as raw" $ \(v :: Value) ->
98+ let res = typeAnnotatedValue <$> (parseEither parseJSON (toJSON (TypeAnnotated v)))
99+ in res === Right v
100+ ]
101+
102+
103+-------------------------------------------------------------------------------
104+exampleValue :: Value
105+exampleValue = Array $ V.fromList [Null, Object ob]
106+ where
107+ ob = HM.fromList [ ("a bool", Bool False)
108+ , ("a long", Number 24)
109+ , ("a double", Number 52.3)
110+ , ("a string", String "s")
111+ , ("a null", Null)
112+ , ("a map", Object (HM.singleton "baz" (Bool True)))
113+ ]
114+
115+
116+-------------------------------------------------------------------------------
117+annotatedExampleValue :: Value
118+annotatedExampleValue = Array $ V.fromList
119+ [
120+ Null
121+ , Object $ HM.fromList
122+ [
123+ ("a map",Object $ HM.fromList [("baz::boolean", Bool True)])
124+ , ("a bool::boolean", Bool False)
125+ , ("a null::null", Null)
126+ , ("a string::string", String "s")
127+ , ("a double::double", Number 52.3)
128+ , ("a long::long", Number 24.0)
129+ ]
61130 ]
62131
63132
@@ -126,3 +195,21 @@ createESSchema :: BH IO Reply
126195createESSchema = do
127196 createIndex ixs ixn
128197 putMapping ixn mn mapping
198+
199+
200+-------------------------------------------------------------------------------
201+instance Arbitrary Value where
202+ arbitrary = oneof
203+ [ Object <$> reduceSize arbitrary
204+ , Array . V.fromList <$> reduceSize arbitrary
205+ , String <$> arbitrary
206+ , Number <$> (scientific <$> arbitrary <*> arbitrary)
207+ , Bool <$> arbitrary
208+ , pure Null
209+ ]
210+
211+
212+-------------------------------------------------------------------------------
213+-- | Reduce the size of Arbitrary input for the given generator
214+reduceSize :: Gen a -> Gen a
215+reduceSize f = sized $ \ n -> resize (n `div` 2) f
katip/src/Katip/Core.hs+2−1
@@ -1,3 +1,4 @@
1+{-# LANGUAGE DeriveFunctor #-}
12{-# LANGUAGE DeriveGeneric #-}
23{-# LANGUAGE ExistentialQuantification #-}
34{-# LANGUAGE FlexibleContexts #-}
@@ -198,7 +199,7 @@ data Item a = Item {
198199 , _itemTime :: UTCTime
199200 , _itemNamespace :: Namespace
200201 , _itemLoc :: Maybe Loc
201− } deriving (Generic)
202+ } deriving (Generic, Functor)
202203makeLenses ''Item
203204
204205