on master help sign in
.gitignore+1
since splitsnapshots
unifiedsplit
@@ -7,3 +7,4 @@ TAGS
7*.hi
7*.hi
8*.dyn_o
8*.dyn_o
9*.dyn_hi
9*.dyn_hi
10+stack.yaml.lock
.travis.yml+10−10
@@ -9,21 +9,19 @@ addons:
9 apt:
9 apt:
10 packages:
10 packages:
11 - libgmp-dev
11 - libgmp-dev
12− - oracle-java8-installer
13
12
14# The different configurations we want to test. You could also do things like
13# The different configurations we want to test. You could also do things like
15# change flags or use --stack-yaml to point to a different file.
14# change flags or use --stack-yaml to point to a different file.
16matrix:
15matrix:
17 include:
16 include:
18− - env: ARGS="--stack-yaml=stack-lts-2.yaml" ESVERSION=1.5.2 ESURL=https://download.elastic.co/elasticsearch/elasticsearch ESTESTARGS="--es-version 1"
17+ - env: ARGS="--resolver=lts-16" ESVERSION=5.5.0 ESURL=https://artifacts.elastic.co/downloads/elasticsearch ESTESTARGS="--es-version 5" ES_JAVA_OPTS="-Xms500m -Xmx750m"
19− - env: ARGS="--stack-yaml=stack-lts-3.yaml" ESVERSION=1.5.2 ESURL=https://download.elastic.co/elasticsearch/elasticsearch ESTESTARGS="--es-version 1"
20− - env: ARGS="--stack-yaml=stack-lts-4.yaml" ESVERSION=1.5.2 ESURL=https://download.elastic.co/elasticsearch/elasticsearch ESTESTARGS="--es-version 1"
18+ - env: ARGS="--resolver=lts-16" ESVERSION=6.2.2 ESURL=https://artifacts.elastic.co/downloads/elasticsearch ESTESTARGS="--es-version 5" ES_JAVA_OPTS="-Xms500m -Xmx750m"
21− - env: ARGS="--stack-yaml=stack-lts-5.yaml" ESVERSION=1.5.2 ESURL=https://download.elastic.co/elasticsearch/elasticsearch ESTESTARGS="--es-version 1"
22− - env: ARGS="--resolver=lts-6" ESVERSION=1.5.2 ESURL=https://download.elastic.co/elasticsearch/elasticsearch ESTESTARGS="--es-version 1"
19+ - env: ARGS="--resolver=nightly" ESVERSION=5.5.0 ESURL=https://artifacts.elastic.co/downloads/elasticsearch ESTESTARGS="--es-version 5" ES_JAVA_OPTS="-Xms500m -Xmx750m"
23− - env: ARGS="--resolver=lts-7" ESVERSION=1.5.2 ESURL=https://download.elastic.co/elasticsearch/elasticsearch ESTESTARGS="--es-version 1"
20+ - env: ARGS="--resolver=nightly" ESVERSION=6.2.2 ESURL=https://artifacts.elastic.co/downloads/elasticsearch ESTESTARGS="--es-version 5" ES_JAVA_OPTS="-Xms500m -Xmx750m"
24− - env: ARGS="--resolver=lts-8" ESVERSION=1.5.2 ESURL=https://download.elastic.co/elasticsearch/elasticsearch ESTESTARGS="--es-version 1"
21+ # nightlies may fail usually due to lagging dependencies
22+ allow_failures:
25− - env: ARGS="--resolver=nightly" ESVERSION=1.5.2 ESURL=https://download.elastic.co/elasticsearch/elasticsearch ESTESTARGS="--es-version 1"
23+ - env: ARGS="--resolver=nightly" ESVERSION=5.5.0 ESURL=https://artifacts.elastic.co/downloads/elasticsearch ESTESTARGS="--es-version 5" ES_JAVA_OPTS="-Xms500m -Xmx750m"
26− - env: ARGS="--resolver=nightly" ESVERSION=5.5.0 ESURL=https://artifacts.elastic.co/downloads/elasticsearch ESTESTARGS="--es-version 5" ES_JAVA_OPTS="-Xms500m -Xmx750m" JAVA_HOME=/usr/lib/jvm/java-8-oracle
24+ - env: ARGS="--resolver=nightly" ESVERSION=6.2.2 ESURL=https://artifacts.elastic.co/downloads/elasticsearch ESTESTARGS="--es-version 5" ES_JAVA_OPTS="-Xms500m -Xmx750m"
27
25
28before_install:
26before_install:
29# Download and unpack the stack executable
27# Download and unpack the stack executable
@@ -36,6 +34,7 @@ install:
36 - tar xzf elasticsearch-$ESVERSION.tar.gz
34 - tar xzf elasticsearch-$ESVERSION.tar.gz
37 # set up a repo for snapshot testing. Required in ES >= 1.6
35 # set up a repo for snapshot testing. Required in ES >= 1.6
38 - echo "path.repo = [\"/tmp\"]" >> ./elasticsearch-$ESVERSION/elasticsearch.yml
36 - echo "path.repo = [\"/tmp\"]" >> ./elasticsearch-$ESVERSION/elasticsearch.yml
37+ - java -version
39 - ./elasticsearch-$ESVERSION/bin/elasticsearch &
38 - ./elasticsearch-$ESVERSION/bin/elasticsearch &
40 # Bloodhound seems to take a while to build and can sometimes stall out the build when travis detects no output.
39 # Bloodhound seems to take a while to build and can sometimes stall out the build when travis detects no output.
41
40
@@ -59,3 +58,4 @@ script:
59cache:
58cache:
60 directories:
59 directories:
61 - $HOME/.stack
60 - $HOME/.stack
61+ - .stack-work
katip-datadog/LICENSEnew file+30
@@ -0,0 +1,30 @@
1+Copyright (c) 2018, Soostone Inc
2+
3+All rights reserved.
4+
5+Redistribution and use in source and binary forms, with or without
6+modification, are permitted provided that the following conditions are met:
7+
8+ * Redistributions of source code must retain the above copyright
9+ notice, this list of conditions and the following disclaimer.
10+
11+ * Redistributions in binary form must reproduce the above
12+ copyright notice, this list of conditions and the following
13+ disclaimer in the documentation and/or other materials provided
14+ with the distribution.
15+
16+ * Neither the name of Ozgun Ataman nor the names of other
17+ contributors may be used to endorse or promote products derived
18+ from this software without specific prior written permission.
19+
20+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
21+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
22+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
23+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
24+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
25+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
26+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
27+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
28+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
29+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
30+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
katip-datadog/README.mdnew file+6
@@ -0,0 +1,6 @@
1+# Katip Datadog [![Build Status](https://travis-ci.org/Soostone/katip.svg?branch=master)](https://travis-ci.org/Soostone/katip)
2+
3+katip-datadog is a scribe for the Katip logging framework that sends
4+structured logs to the Datadog logging service. It supports both
5+directly sending to the ingestion API and through a locally installed
6+datadog agent.
katip-datadog/changelog.mdnew file+7
@@ -0,0 +1,7 @@
1+0.2.0.0
2+==========
3+* Remove import of deprecated Network module
4+
5+0.1.0.0
6+==========
7+* Initial release
katip-datadog/katip-datadog.cabalnew file+78
@@ -0,0 +1,78 @@
1+name: katip-datadog
2+version: 0.2.0.0
3+synopsis: Datadog scribe for the Katip logging framework
4+description: See README.md for more details.
5+license: BSD3
6+license-file: LICENSE
7+author: Michael Xavier
8+maintainer: michael.xavier@soostone.com
9+copyright: Soostone Inc, 2018
10+category: Data, Text, Logging
11+homepage: https://github.com/Soostone/katip
12+bug-reports: https://github.com/Soostone/katip/issues
13+build-type: Simple
14+cabal-version: >=1.10
15+extra-source-files:
16+ README.md
17+ changelog.md
18+
19+--TODO: fill this in
20+tested-with: GHC == 8.4.3
21+
22+source-repository head
23+ type: git
24+ location: https://github.com/Soostone/katip.git
25+
26+flag lib-Werror
27+ default: False
28+ manual: True
29+
30+
31+library
32+ exposed-modules:
33+ Katip.Scribes.Datadog.TCP
34+ build-depends:
35+ base >=4.4.0.0 && <5
36+ , aeson >= 1.0.0.0
37+ , bytestring
38+ , katip >= 0.8.0.0
39+ , text
40+ , connection
41+ , resource-pool
42+ , time
43+ , binary
44+ , network
45+ , safe-exceptions
46+ , retry >= 0.7
47+ hs-source-dirs: src
48+ default-language: Haskell2010
49+ hs-source-dirs: src
50+ ghc-options: -Wall
51+ if flag(lib-Werror)
52+ ghc-options: -Werror
53+
54+
55+test-suite test
56+ type: exitcode-stdio-1.0
57+ main-is: Main.hs
58+ hs-source-dirs: test
59+ default-language: Haskell2010
60+ other-modules:
61+ Katip.Tests.Scribes.Datadog.TCP
62+ build-depends: base
63+ , katip
64+ , katip-datadog
65+ , conduit
66+ , conduit-extra
67+ , tasty
68+ , tasty-hunit
69+ , aeson
70+ , unordered-containers
71+ , containers
72+ , safe-exceptions
73+ , async
74+ , attoparsec
75+ , text
76+ ghc-options: -Wall
77+ if flag(lib-Werror)
78+ ghc-options: -Werror
katip-datadog/src/Katip/Scribes/Datadog/TCP.hsnew file+191
@@ -0,0 +1,191 @@
1+{-# LANGUAGE BangPatterns #-}
2+{-# LANGUAGE OverloadedStrings #-}
3+{-# LANGUAGE RankNTypes #-}
4+-- | Creates a scribe as a Custom Forwarder for Datadog that sends log
5+-- messages over TCP via their TLS endpoint.
6+module Katip.Scribes.Datadog.TCP
7+ ( -- * Types
8+ APIKey(..)
9+ , DatadogScribeSettings(..)
10+ , DatadogAuth(..)
11+ , directAPIConnectionParams
12+ , localAgentConnectionParams
13+ -- * Scribe construction
14+ , mkDatadogScribeSettings
15+ , mkDatadogScribe
16+ ) where
17+
18+
19+
20+-------------------------------------------------------------------------------
21+import qualified Control.Concurrent as Conc
22+import qualified Control.Exception.Safe as EX
23+import Control.Monad (void)
24+import qualified Control.Retry as Retry
25+import qualified Data.Aeson as A
26+import qualified Data.Binary.Builder as BB
27+import qualified Data.ByteString.Lazy as BSL
28+import Data.Monoid as Monoid
29+import qualified Data.Pool as Pool
30+import Data.Text (Text)
31+import qualified Data.Text.Encoding as T
32+import qualified Data.Text.Lazy.Builder as B
33+import Data.Time (NominalDiffTime)
34+import Katip
35+import Katip.Core (LocJs (..))
36+import qualified Network.Connection as C
37+import qualified Network.Socket as Net
38+import qualified System.Posix.Types as POSIX
39+-------------------------------------------------------------------------------
40+
41+
42+-- | When writing directly to the main intake API, you must specify an
43+-- API key
44+newtype APIKey = APIKey
45+ { apiKey :: Text
46+ } deriving (Show, Eq)
47+
48+
49+data DatadogAuth =
50+ NoAuthLocal
51+ -- ^ Writing to the local agent, no auth required
52+ | DirectAuth APIKey
53+ -- ^ Writing directly to the main intake API over TLS . API Key is
54+ -- required. Using a local agent is recommended.
55+
56+
57+data DatadogScribeSettings = DatadogScribeSettings
58+ { datadogScribeSettings_connectionParams :: C.ConnectionParams
59+ -- ^ Specify where the logs should go. If writing directly to the
60+ -- main intake API, you can use 'directAPIConnectionParams'. If
61+ -- writing to a local agent, we recommend
62+ -- 'localAgentConnectionParams'
63+ , datadogScribeSettings_auth :: DatadogAuth
64+ , datadogScribeSettings_poolStripes :: Int
65+ -- ^ How many stripes should be used in the connection pool. 1 is a reasonable default
66+ , datadogScribeSettings_connsPerStripe :: Int
67+ -- ^ How many TCP connections per stripe should be maintained in the pool.
68+ , datadogScribeSettings_connectionKeepalive :: NominalDiffTime
69+ -- ^ How long should a TCP connection be kept open. 30 (seconds) is a reasonable default. Anything around 60 will tend to time out a lot.
70+ , datadogScribeSettings_retry :: Retry.RetryPolicyM IO
71+ -- ^ How should exceptions during write be retried?
72+ }
73+
74+
75+-- | Configured for secure TLS communication to the main intake API at
76+-- intake.logs.datadoghq.com:10516. Must be combined with DirectAuth
77+-- authentication.
78+directAPIConnectionParams :: C.ConnectionParams
79+directAPIConnectionParams = C.ConnectionParams
80+ { C.connectionHostname = "intake.logs.datadoghq.com"
81+ , C.connectionPort = 10516
82+ , C.connectionUseSecure = Just $ C.TLSSettingsSimple
83+ { C.settingDisableCertificateValidation = False
84+ , C.settingDisableSession = False
85+ , C.settingUseServerName = False
86+ }
87+ , C.connectionUseSocks = Nothing
88+ }
89+
90+
91+-- | Configure to talk to a local Datadog agent at 127.0.0.1. TLS is not
92+-- utilized. This is the preferred type of connection because
93+-- authentication, buffering, and other more advanced features are
94+-- handled by the agent configuration.
95+localAgentConnectionParams :: Net.PortNumber -> C.ConnectionParams
96+localAgentConnectionParams port = C.ConnectionParams
97+ { C.connectionHostname = "127.0.0.1"
98+ , C.connectionPort = port
99+ , C.connectionUseSecure = Nothing
100+ , C.connectionUseSocks = Nothing
101+ }
102+
103+
104+-- | Reasonable defaults. Sets a 30s connection timeout, 1 stripe of
105+-- connections with a number of connections per stripe equal to
106+-- 'getNumCapabilities'. Retry policy will do an exponential backoff
107+-- with a 25ms base delay for up to 5 retries for a total cumulative delay of 775ms
108+mkDatadogScribeSettings :: C.ConnectionParams -> DatadogAuth -> IO DatadogScribeSettings
109+mkDatadogScribeSettings connectionParams auth = do
110+ capabilities <- Conc.getNumCapabilities
111+ pure $ DatadogScribeSettings
112+ { datadogScribeSettings_connectionParams = connectionParams
113+ , datadogScribeSettings_auth = auth
114+ , datadogScribeSettings_poolStripes = 1
115+ , datadogScribeSettings_connsPerStripe = capabilities
116+ , datadogScribeSettings_connectionKeepalive = 30
117+ , datadogScribeSettings_retry = Retry.exponentialBackoff 25000 <> Retry.limitRetries 5
118+ }
119+
120+
121+-------------------------------------------------------------------------------
122+mkDatadogScribe
123+ :: DatadogScribeSettings
124+ -> PermitFunc
125+ -- ^ Function on whether to permit items, e.g. @permitItem InfoS@
126+ -> Verbosity
127+ -- ^ Verbosity level to observe
128+ -> IO Scribe
129+mkDatadogScribe settings permit verb = do
130+ connectionContext <- C.initConnectionContext
131+ pool <- Pool.createPool
132+ (C.connectTo connectionContext (datadogScribeSettings_connectionParams settings))
133+ C.connectionClose
134+ (datadogScribeSettings_poolStripes settings)
135+ (datadogScribeSettings_connectionKeepalive settings)
136+ (datadogScribeSettings_connsPerStripe settings)
137+ pure $ Scribe
138+ { liPush = pushPool (datadogScribeSettings_retry settings) pool keyBuilder verb
139+ , scribeFinalizer = Pool.destroyAllResources pool
140+ , scribePermitItem = permit
141+ }
142+ where
143+ !keyBuilder = case datadogScribeSettings_auth settings of
144+ NoAuthLocal -> Nothing
145+ DirectAuth (APIKey k) -> Just (BB.fromByteString (T.encodeUtf8 k))
146+
147+
148+pushPool
149+ :: LogItem a
150+ => Retry.RetryPolicyM IO
151+ -> Pool.Pool C.Connection
152+ -> Maybe BB.Builder
153+ -- ^ Rendered API key
154+ -> Verbosity
155+ -> Item a
156+ -> IO ()
157+pushPool retryPolicy pool token verb item =
158+ void $ Retry.retrying retryPolicy (\_status shouldRetry -> pure shouldRetry) $ \_status -> do
159+ res <- EX.tryAny $ Pool.withResource pool $ \conn -> do
160+ C.connectionPut conn rendered
161+ pure $ case res of
162+ Left _ -> True
163+ Right _ -> False
164+ where
165+ payloadBuilder = A.fromEncoding (encodeDatadog verb item) Monoid.<> "\n"
166+ messageBuilder = case token of
167+ --TODO: lots of conversion going on here. could use a bench
168+ Nothing -> payloadBuilder
169+ Just directKey -> directKey <> " " <> payloadBuilder
170+ rendered = BSL.toStrict (BB.toLazyByteString messageBuilder)
171+
172+
173+encodeDatadog :: LogItem a => Verbosity -> Item a -> A.Encoding
174+encodeDatadog verb i = A.pairs $
175+ -- datadog seems to use "service" to denote appname
176+ "service" A..= _itemApp i <>
177+ -- datadog uses severity (or some other alternatives for sev)
178+ "severity" A..= _itemSeverity i <>
179+ "env" A..= _itemEnv i <>
180+ "thread" A..= getThreadIdText (_itemThread i) <>
181+ "host" A..= _itemHost i <>
182+ "pid" A..= pidInt <>
183+ "data" A..= payloadObject verb (_itemPayload i) <>
184+ -- datadog uses "message" instead of msg
185+ "message" A..= (B.toLazyText (unLogStr (_itemMessage i))) <>
186+ -- datadog uses "date" instead of "at"
187+ "date" A..= _itemTime i <>
188+ "ns" A..= _itemNamespace i <>
189+ "loc" A..= fmap LocJs (_itemLoc i)
190+ where
191+ POSIX.CPid pidInt = _itemProcess i
katip-datadog/test/Katip/Tests/Scribes/Datadog/TCP.hsnew file+105
@@ -0,0 +1,105 @@
1+{-# LANGUAGE OverloadedStrings #-}
2+module Katip.Tests.Scribes.Datadog.TCP
3+ ( tests
4+ ) where
5+
6+
7+-------------------------------------------------------------------------------
8+import Control.Concurrent.Async
9+import qualified Control.Exception.Safe as EX
10+import qualified Data.Aeson as A
11+import qualified Data.Char as Char
12+import qualified Data.Conduit as C
13+import qualified Data.Conduit.Binary as CB
14+import qualified Data.Conduit.List as CL
15+import qualified Data.Conduit.Network as CN
16+import Data.Foldable (toList)
17+import qualified Data.HashMap.Strict as HM
18+import Data.IORef
19+import Data.Sequence ((|>))
20+import Data.Text (Text)
21+import qualified Data.Text as T
22+import Test.Tasty
23+import Test.Tasty.HUnit
24+-------------------------------------------------------------------------------
25+import Katip
26+import Katip.Scribes.Datadog.TCP
27+-------------------------------------------------------------------------------
28+
29+
30+
31+tests :: TestTree
32+tests = testGroup "Katip.Scribes.Datadog.TCP"
33+ [ testCase "logs well-formed messages in order" $ do
34+ logs <- collectLogs $ do
35+ logItem (sl "foo" ("bar" :: Text)) "mynamespace" noLoc InfoS "a message"
36+ length logs @?= 1
37+ let (CapturedLog ns dta sev msg thread) = head logs
38+ ns @?= Namespace ["katip-datadog-tests", "mynamespace"]
39+ dta @?= A.Object (HM.singleton "foo" (A.String "bar"))
40+ sev @?= InfoS
41+ msg @?= "a message"
42+ if T.all Char.isNumber thread && T.length thread > 0
43+ then pure ()
44+ else assertFailure ("Expected thread id to just be a number but it was " <> T.unpack thread)
45+ ]
46+ where
47+ noLoc = Nothing
48+
49+
50+-------------------------------------------------------------------------------
51+-- | A log type parsed out of the fake datadog agent's buffer
52+data CapturedLog = CapturedLog
53+ { capturedLog_namespace :: Namespace
54+ , capturedLog_data :: A.Value
55+ , capturedLog_sev :: Severity
56+ , capturedLog_message :: Text
57+ , capturedLog_thread :: Text
58+ } deriving (Show, Eq)
59+
60+
61+instance A.FromJSON CapturedLog where
62+ parseJSON = A.withObject "CapturedLog" $ \o -> do
63+ message <- o A..: "message"
64+ sev <- o A..: "severity"
65+ dta <- o A..: "data"
66+ namespace <- o A..: "ns"
67+ thread <- o A..: "thread"
68+ pure $ CapturedLog
69+ { capturedLog_namespace = namespace
70+ , capturedLog_data = dta
71+ , capturedLog_sev = sev
72+ , capturedLog_message = message
73+ , capturedLog_thread = thread
74+ }
75+
76+
77+-------------------------------------------------------------------------------
78+collectLogs :: KatipT IO () -> IO [CapturedLog]
79+collectLogs f = do
80+ capturedRef <- newIORef mempty
81+ withAsync (startServer capturedRef) $ \_worker -> do
82+ EX.bracket mkLE closeLE $ \le -> do
83+ runKatipT le f
84+ toList <$> readIORef capturedRef
85+ where
86+ closeLE = closeScribes
87+ port = 1337
88+ mkLE = do
89+ datadogScribeSettings <- mkDatadogScribeSettings (localAgentConnectionParams (fromInteger port)) NoAuthLocal
90+ scribe <- mkDatadogScribe datadogScribeSettings (permitItem DebugS) V3
91+ registerScribe "datadog" scribe defaultScribeSettings =<< initLogEnv "katip-datadog-tests" "test"
92+ startServer capturedRef = CN.runTCPServer (CN.serverSettings (fromInteger port) "*") $ \appData -> C.runConduit $
93+ CN.appSource appData C..|
94+ CB.lines C..|
95+ CL.map A.eitherDecodeStrict C..|
96+ CL.mapM_ (\parseResult -> case parseResult of
97+ Left e -> EX.throwM (ParseError e)
98+ (Right capturedLog) -> modifyIORef' capturedRef (\captured -> captured |> capturedLog))
99+
100+
101+-------------------------------------------------------------------------------
102+newtype ParseError = ParseError String
103+ deriving (Show)
104+
105+instance EX.Exception ParseError
katip-datadog/test/Main.hsnew file+22
@@ -0,0 +1,22 @@
1+module Main
2+ ( main
3+ ) where
4+
5+
6+-------------------------------------------------------------------------------
7+import Test.Tasty
8+-------------------------------------------------------------------------------
9+import qualified Katip.Tests.Scribes.Datadog.TCP
10+-------------------------------------------------------------------------------
11+
12+
13+
14+main :: IO ()
15+main = defaultMain tests
16+
17+
18+-------------------------------------------------------------------------------
19+tests :: TestTree
20+tests = testGroup "katip-datadog"
21+ [ Katip.Tests.Scribes.Datadog.TCP.tests
22+ ]
katip-elasticsearch/bench/Main.hs+12−15
@@ -13,30 +13,29 @@ import Criterion.Main
13import Data.Aeson
13import Data.Aeson
14import qualified Data.HashMap.Strict as HM
14import qualified Data.HashMap.Strict as HM
15import Data.Proxy (Proxy (..))
15import Data.Proxy (Proxy (..))
16−import Data.RNG
16+import System.Random
17import qualified Data.Text as T
17import qualified Data.Text as T
18−import Database.V1.Bloodhound.Types
18+import Database.Bloodhound.Types
19import Numeric
19import Numeric
20-------------------------------------------------------------------------------
20-------------------------------------------------------------------------------
21import Katip.Scribes.ElasticSearch
21import Katip.Scribes.ElasticSearch
22−import Katip.Scribes.ElasticSearch.Internal (ESV1)
22+import Katip.Scribes.ElasticSearch.Internal (ESV5)
23-------------------------------------------------------------------------------
23-------------------------------------------------------------------------------
24
24
25main :: IO ()
25main :: IO ()
26main = do
26main = do
27− rng <- mkRNG
28 defaultMain
27 defaultMain
29 [
28 [
30− mkDocIdBenchmark rng
29+ mkDocIdBenchmark
31 , deannotateValueBenchmark
30 , deannotateValueBenchmark
32 ]
31 ]
33
32
34-------------------------------------------------------------------------------
33-------------------------------------------------------------------------------
35−mkDocIdBenchmark :: RNG -> Benchmark
34+mkDocIdBenchmark :: Benchmark
36−mkDocIdBenchmark rng = bgroup "mkDocId"
35+mkDocIdBenchmark = bgroup "mkDocId"
37 [
36 [
38− bench "mkDocId (randomIO)" $ nfIO (mkDocId (Proxy :: Proxy ESV1))
37+ bench "mkDocId (randomIO)" $ nfIO (mkDocId (Proxy :: Proxy ESV5))
39− , bench "mkDocId' (shared )" $ nfIO $ mkDocId' rng
38+ , bench "mkDocId' (shared )" $ nfIO mkDocId'
40 ]
39 ]
41
40
42
41
@@ -58,13 +57,11 @@ annotatedValue = Object $ HM.fromList [ ("a::string", String "whatever")
58 ]
57 ]
59
58
60-------------------------------------------------------------------------------
59-------------------------------------------------------------------------------
61−mkDocId' :: RNG -> IO DocId
60+mkDocId' :: IO DocId
62−mkDocId' rng = do
61+mkDocId' = do
63− is <- withRNG rng $ \gen -> replicateM len $ mk gen
62+ is <- replicateM len (randomRIO (0, 15))
64− return . DocId . T.pack . concatMap (`showHex` "") $ is
63+ return (DocId (T.pack (concatMap (`showHex` "") (is :: [Int]))))
65 where
64 where
66 len = 32
65 len = 32
67− mk :: GenIO -> IO Int
68− mk = uniformR (0,15)
69
66
70deriving instance NFData DocId
67deriving instance NFData DocId
katip-elasticsearch/changelog.md+60
@@ -1,3 +1,63 @@
1+0.7.0.0
2+=======
3+* Make compatible with unreleased bloodhound 0.17
4+* Deprecate use of ES version 1 which has been removed from bloodhound
5+ 0.17. All users should move over to ESV5 which should work for
6+ versions 5 and up.
7+* Default configuration to ESV5
8+* Add deprecation warnings for ESV1. Support for ESV1 will be removed
9+ in the future.
10+
11+0.6.0.0
12+=======
13+* Update to katip-0.8.0.0 `PermitFunc` convention. To update, for
14+ example, if you were passing `InfoS` to `mkEsScribe` to filter
15+ severities, instead you should pass `(permitItem InfoS)`.
16+* Update examples to add necessary type signature for the default
17+ scribe config.
18+* Bugfix: the default error retry policy was much more rapid than
19+ documented.
20+* Internally switch to safe-exceptions from encolosed-exceptions.
21+
22+0.5.1.1
23+=======
24+* Loosen katip dep
25+
26+0.5.1.0
27+=======
28+* Loosen deps
29+
30+0.5.0.0
31+=======
32+* Update template for ESv1 to not analyze certain fields like host and
33+ namespace. These are not fields that benefit from tokenization. If
34+ you're on ESv1, after running `mkEsScribe`, the logs template will
35+ be updated to stop analyzing those fields. That means the next
36+ index that is cut (e.g. tomorrow's logs if you're using daily index
37+ sharding) will no longer analyze some fields.
38+
39+0.4.2.0
40+=======
41+* Allow newer bloodhound, aeson, async.
42+
43+0.4.1.0
44+=======
45+* Fix bug where index was created in `mkEsScribe` when it would not be used due to index sharding.
46+* Update some index settings if index already exists and sharding is not used.
47+* For ES V5 and higher, stop using the deprecated (and in 6.x, removed) `string` type for index templates, instead using `text` and `keyword` as appropriate. This makes `katip-elasticsearch` compatible with ES 6.x.
48+
49+0.4.0.4
50+=======
51+* Allow http-types 0.12
52+
53+0.4.0.3
54+=======
55+* Fix benchmark and test builds for stackage nightly
56+
57+0.4.0.2
58+=======
59+* Add repository/homepage info to cabal file
60+
10.4.0.1
610.4.0.1
2=======
62=======
3* Bump dependencies to allow GHC 8.2.1
63* Bump dependencies to allow GHC 8.2.1
katip-elasticsearch/docker-compose.ymlnew file+10
@@ -0,0 +1,10 @@
1+version: "3.7"
2+services:
3+ # https://hub.docker.com/_/elasticsearch
4+ elasticsearch:
5+ image: elasticsearch:5-alpine
6+ ports:
7+ - target: 9200
8+ published: 9200
9+ protocol: tcp
10+ mode: host
katip-elasticsearch/examples/example.hs+5−4
@@ -6,7 +6,7 @@ module Main
6
6
7-------------------------------------------------------------------------------
7-------------------------------------------------------------------------------
8import Control.Exception
8import Control.Exception
9−import Database.V5.Bloodhound
9+import Database.Bloodhound
10import Network.HTTP.Client
10import Network.HTTP.Client
11-------------------------------------------------------------------------------
11-------------------------------------------------------------------------------
12import Katip
12import Katip
@@ -17,16 +17,17 @@ import Katip.Scribes.ElasticSearch
17main :: IO ()
17main :: IO ()
18main = do
18main = do
19 mgr <- newManager defaultManagerSettings
19 mgr <- newManager defaultManagerSettings
20− let bhe = mkBHEnv (Server "localhost") mgr
20+ let bhe = mkBHEnv (Server "http://localhost:9200") mgr
21+ -- You may need to specify the version in a type signature when customizing your index settings
22+ let _customizedExample = defaultEsScribeCfgV5 { essIndexSettings = IndexSettings (ShardCount 1) (ReplicaCount 0)} :: EsScribeCfg ESV5
21 esScribe <- mkEsScribe
23 esScribe <- mkEsScribe
22 -- Reasonable for production
24 -- Reasonable for production
23 defaultEsScribeCfgV5
25 defaultEsScribeCfgV5
24 -- Reasonable for single-node in development
26 -- Reasonable for single-node in development
25− -- defaultEsScribeCfgV5 { essIndexSettings = IndexSettings (ShardCound 1) (ReplicaCount 0)}
26 bhe
27 bhe
27 (IndexName "all-indices-prefixed-with")
28 (IndexName "all-indices-prefixed-with")
28 (MappingName "application-logs")
29 (MappingName "application-logs")
29− DebugS
30+ (permitItem DebugS)
30 V3
31 V3
31 let mkLogEnv = registerScribe "es" esScribe defaultScribeSettings =<< initLogEnv "MyApp" "production"
32 let mkLogEnv = registerScribe "es" esScribe defaultScribeSettings =<< initLogEnv "MyApp" "production"
32 bracket mkLogEnv closeScribes $ \le -> runKatipT le $ do
33 bracket mkLogEnv closeScribes $ \le -> runKatipT le $ do
katip-elasticsearch/katip-elasticsearch.cabal+28−25
@@ -1,13 +1,15 @@
1name: katip-elasticsearch
1name: katip-elasticsearch
2synopsis: ElasticSearch scribe for the Katip logging framework.
2synopsis: ElasticSearch scribe for the Katip logging framework.
3description: See README.md for more details.
3description: See README.md for more details.
4−version: 0.4.0.1
4+version: 0.7.0.0
5license: BSD3
5license: BSD3
6license-file: LICENSE
6license-file: LICENSE
7author: Ozgun Ataman, Michael Xavier
7author: Ozgun Ataman, Michael Xavier
8maintainer: michael.xavier@soostone.com
8maintainer: michael.xavier@soostone.com
9−copyright: Soostone Inc, 2015-2016
9+copyright: Soostone Inc, 2015-2017
10−category: Data
10+category: Data, Text, Logging
11+homepage: https://github.com/Soostone/katip
12+bug-reports: https://github.com/Soostone/katip/issues
11build-type: Simple
13build-type: Simple
12cabal-version: >=1.10
14cabal-version: >=1.10
13extra-source-files:
15extra-source-files:
@@ -16,7 +18,11 @@ extra-source-files:
16 bench/Main.hs
18 bench/Main.hs
17 examples/example.hs
19 examples/example.hs
18 test/Main.hs
20 test/Main.hs
19−tested-with: GHC == 7.8.4, GHC== 7.10.3
21+tested-with: GHC == 7.8.4, GHC== 7.10.3, GHC == 8.0.1, GHC == 8.2.2, GHC == 8.4.3
22+
23+source-repository head
24+ type: git
25+ location: https://github.com/Soostone/katip.git
20
26
21flag lib-Werror
27flag lib-Werror
22 default: False
28 default: False
@@ -29,28 +35,28 @@ library
29 Katip.Scribes.ElasticSearch.Internal
35 Katip.Scribes.ElasticSearch.Internal
30 build-depends:
36 build-depends:
31 base >=4.6 && <5
37 base >=4.6 && <5
32− , katip >= 0.2.0.0 && < 0.6
38+ , katip >= 0.8.0.0
33− , bloodhound >= 0.13.0.0 && < 0.16
39+ , bloodhound >= 0.13.0.0
34− , uuid >= 1.3.12 && < 1.4
40+ , uuid >= 1.3.12
35− , aeson >=0.6 && <1.3
41+ , aeson >=0.6
36− , stm >= 2.4.3 && < 2.5
42+ , stm >= 2.4.3
37− , stm-chans >= 3.0.0.2 && < 3.1
43+ , stm-chans >= 3.0.0.2
38− , async >= 2.0.1.0 && < 2.2
44+ , async >= 2.0.1.0
39− , enclosed-exceptions >= 1.0.0 && < 1.1
45+ , safe-exceptions >= 0.1.0.0
40− , http-client >= 0.3 && < 0.6
46+ , http-client >= 0.3
41− , text >= 0.11 && < 1.3
47+ , text >= 0.11
42− , retry >= 0.7 && < 0.8
48+ , retry >= 0.7
43 , exceptions
49 , exceptions
44− , scientific >= 0.3.0.0 && < 0.4
50+ , scientific >= 0.3.0.0
45− , unordered-containers >= 0.1.0.0 && < 0.3
51+ , unordered-containers >= 0.1.0.0
46− , transformers >= 0.2 && < 0.6
52+ , transformers >= 0.2
47− , http-types >= 0.8 && < 0.10
53+ , http-types >= 0.8
48− , time >= 1 && < 1.9
54+ , time >= 1
49 , bytestring
55 , bytestring
56+ , semigroups
50 hs-source-dirs: src
57 hs-source-dirs: src
51 default-language: Haskell2010
58 default-language: Haskell2010
52 hs-source-dirs: src
59 hs-source-dirs: src
53− default-language: Haskell2010
54 if flag(lib-Werror)
60 if flag(lib-Werror)
55 ghc-options: -Wall -Werror
61 ghc-options: -Wall -Werror
56
62
@@ -66,7 +72,7 @@ benchmark bench
66 , bloodhound
72 , bloodhound
67 , deepseq
73 , deepseq
68 , criterion >= 1.1.0.0
74 , criterion >= 1.1.0.0
69− , rng-utils
75+ , random
70 , uuid
76 , uuid
71 , text
77 , text
72 , unordered-containers
78 , unordered-containers
@@ -102,7 +108,4 @@ test-suite test
102 , bytestring
108 , bytestring
103 , tagged
109 , tagged
104
110
105− if flag(lib-Werror)
106− ghc-options: -Wall -Werror
107−
108 ghc-options: -threaded -rtsopts
111 ghc-options: -threaded -rtsopts
katip-elasticsearch/src/Katip/Scribes/ElasticSearch.hs+7−2
@@ -1,3 +1,4 @@
1+{-# LANGUAGE CPP #-}
1-- | Includes a scribe that can be used to log structured, JSON log
2-- | Includes a scribe that can be used to log structured, JSON log
2-- messages to ElasticSearch. These logs can be explored easily using
3-- messages to ElasticSearch. These logs can be explored easily using
3-- <https://www.elastic.co/products/kibana kibana> or your tool of
4-- <https://www.elastic.co/products/kibana kibana> or your tool of
@@ -17,12 +18,12 @@
17-- main :: IO ()
18-- main :: IO ()
18-- main = do
19-- main = do
19-- mgr <- newManager defaultManagerSettings
20-- mgr <- newManager defaultManagerSettings
20−-- let bhe = mkBHEnv (Server "localhost") mgr
21+-- let bhe = mkBHEnv (Server "http://localhost:9200") mgr
21-- esScribe <- mkEsScribe
22-- esScribe <- mkEsScribe
22-- -- Reasonable for production
23-- -- Reasonable for production
23-- defaultEsScribeCfgV5
24-- defaultEsScribeCfgV5
24-- -- Reasonable for single-node in development
25-- -- Reasonable for single-node in development
25−-- -- defaultEsScribeCfgV5 { essIndexSettings = IndexSettings (ShardCound 1) (ReplicaCount 0)}
26+-- -- defaultEsScribeCfgV5 { essIndexSettings = IndexSettings (ShardCound 1) (ReplicaCount 0)} :: EsScribeCfg ESV5
26-- bhe
27-- bhe
27-- (IndexName "all-indices-prefixed-with")
28-- (IndexName "all-indices-prefixed-with")
28-- (MappingName "application-logs")
29-- (MappingName "application-logs")
@@ -67,12 +68,16 @@ module Katip.Scribes.ElasticSearch
67 , essIndexSettings
68 , essIndexSettings
68 , essIndexSharding
69 , essIndexSharding
69 , defaultEsScribeCfg
70 , defaultEsScribeCfg
71+#if !MIN_VERSION_bloodhound(0,17,0)
70 , defaultEsScribeCfgV1
72 , defaultEsScribeCfgV1
73+#endif
71 , defaultEsScribeCfgV5
74 , defaultEsScribeCfgV5
72 -- ** Version-Proxied APIS
75 -- ** Version-Proxied APIS
73 -- $versionproxies
76 -- $versionproxies
74 , defaultEsScribeCfg'
77 , defaultEsScribeCfg'
78+#if !MIN_VERSION_bloodhound(0,17,0)
75 , ESV1
79 , ESV1
80+#endif
76 , ESV5
81 , ESV5
77 -- * Utilities
82 -- * Utilities
78 , mkDocId
83 , mkDocId
katip-elasticsearch/src/Katip/Scribes/ElasticSearch/Annotations.hs+2−2
@@ -12,7 +12,7 @@ import Data.Aeson
12import qualified Data.Foldable as FT
12import qualified Data.Foldable as FT
13import qualified Data.HashMap.Strict as HM
13import qualified Data.HashMap.Strict as HM
14import Data.Maybe
14import Data.Maybe
15−import Data.Monoid
15+import Data.Monoid as M
16import Data.Scientific (isFloating)
16import Data.Scientific (isFloating)
17import Data.Text (Text)
17import Data.Text (Text)
18import qualified Data.Text as T
18import qualified Data.Text as T
@@ -65,7 +65,7 @@ annotateKeys = HM.fromList . map go . HM.toList
65 where
65 where
66 go (k, Object o) = (k, Object $ annotateKeys o)
66 go (k, Object o) = (k, Object $ annotateKeys o)
67 go (k, Array a) = (k, Array (annotateValue <$> a))
67 go (k, Array a) = (k, Array (annotateValue <$> a))
68− go (k, s@(String _)) = (k <> stringAnn, s)
68+ go (k, s@(String _)) = (k M.<> stringAnn, s)
69 go (k, n@(Number sci)) = if isFloating sci
69 go (k, n@(Number sci)) = if isFloating sci
70 then (k <> doubleAnn, n)
70 then (k <> doubleAnn, n)
71 else (k <> longAnn, n)
71 else (k <> longAnn, n)
katip-elasticsearch/src/Katip/Scribes/ElasticSearch/Internal.hs+101−44
@@ -16,8 +16,7 @@ import Control.Applicative as A
16import Control.Concurrent
16import Control.Concurrent
17import Control.Concurrent.Async
17import Control.Concurrent.Async
18import Control.Concurrent.STM.TBMQueue
18import Control.Concurrent.STM.TBMQueue
19−import Control.Exception.Base
19+import qualified Control.Exception.Safe as EX
20−import Control.Exception.Enclosed
21import Control.Monad
20import Control.Monad
22import Control.Monad.Catch
21import Control.Monad.Catch
23import Control.Monad.IO.Class
22import Control.Monad.IO.Class
@@ -28,7 +27,8 @@ import Control.Retry (RetryPolicy,
28 recovering)
27 recovering)
29import Data.Aeson
28import Data.Aeson
30import Data.ByteString.Lazy (ByteString)
29import Data.ByteString.Lazy (ByteString)
31−import Data.Monoid ((<>))
30+import Data.List.NonEmpty (NonEmpty (..))
31+import Data.Monoid as Monoid
32import Data.Text (Text)
32import Data.Text (Text)
33import qualified Data.Text as T
33import qualified Data.Text as T
34import qualified Data.Text.Encoding as T
34import qualified Data.Text.Encoding as T
@@ -37,8 +37,12 @@ import Data.Time.Calendar.WeekDate
37import Data.Typeable as Typeable
37import Data.Typeable as Typeable
38import Data.UUID
38import Data.UUID
39import qualified Data.UUID.V4 as UUID4
39import qualified Data.UUID.V4 as UUID4
40+#if MIN_VERSION_bloodhound(0,17,0)
41+import qualified Database.Bloodhound as V5
42+#else
40import qualified Database.V1.Bloodhound as V1
43import qualified Database.V1.Bloodhound as V1
41import qualified Database.V5.Bloodhound as V5
44import qualified Database.V5.Bloodhound as V5
45+#endif
42import Network.HTTP.Client
46import Network.HTTP.Client
43import Network.HTTP.Types.Status
47import Network.HTTP.Types.Status
44import Text.Printf (printf)
48import Text.Printf (printf)
@@ -84,7 +88,7 @@ data EsScribeCfg v = EsScribeCfg {
84--
88--
85-- * defaultManagerSettings
89-- * defaultManagerSettings
86--
90--
87−-- * exponential backoff with 25ms base delay up to 5 retries
91+-- * exponential backoff with 25ms base delay up to 5 retries, for a total cumulative delay of 775ms
88--
92--
89-- * Queue size of 1000
93-- * Queue size of 1000
90--
94--
@@ -95,7 +99,7 @@ data EsScribeCfg v = EsScribeCfg {
95-- * DailyIndexSharding
99-- * DailyIndexSharding
96defaultEsScribeCfg' :: ESVersion v => proxy v -> EsScribeCfg v
100defaultEsScribeCfg' :: ESVersion v => proxy v -> EsScribeCfg v
97defaultEsScribeCfg' prx = EsScribeCfg {
101defaultEsScribeCfg' prx = EsScribeCfg {
98− essRetryPolicy = exponentialBackoff 25 <> limitRetries 5
102+ essRetryPolicy = exponentialBackoff 25000 Monoid.<> limitRetries 5
99 , essQueueSize = EsQueueSize 1000
103 , essQueueSize = EsQueueSize 1000
100 , essPoolSize = EsPoolSize 2
104 , essPoolSize = EsPoolSize 2
101 , essAnnotateTypes = False
105 , essAnnotateTypes = False
@@ -104,19 +108,22 @@ defaultEsScribeCfg' prx = EsScribeCfg {
104 }
108 }
105
109
106
110
111+
112+
107-------------------------------------------------------------------------------
113-------------------------------------------------------------------------------
108−-- | Alias of 'defaultEsScribeCfgV1' to minimize API
114+-- | Alias of 'defaultEsScribeCfgV5' to minimize API
109-- breakage. Previous versions of katip-elasticsearch only supported
115-- breakage. Previous versions of katip-elasticsearch only supported
110−-- ES version 1.
116+-- ES version 1 and defaulted to it.
111−defaultEsScribeCfg :: EsScribeCfg ESV1
117+defaultEsScribeCfg :: EsScribeCfg ESV5
112−defaultEsScribeCfg = defaultEsScribeCfgV1
118+defaultEsScribeCfg = defaultEsScribeCfgV5
113
119
114
120
115-------------------------------------------------------------------------------
121-------------------------------------------------------------------------------
122+#if !MIN_VERSION_bloodhound(0,17,0)
116-- | EsScribeCfg that will use ElasticSearch V1
123-- | EsScribeCfg that will use ElasticSearch V1
117defaultEsScribeCfgV1 :: EsScribeCfg ESV1
124defaultEsScribeCfgV1 :: EsScribeCfg ESV1
118defaultEsScribeCfgV1 = defaultEsScribeCfg' (Typeable.Proxy :: Typeable.Proxy ESV1)
125defaultEsScribeCfgV1 = defaultEsScribeCfg' (Typeable.Proxy :: Typeable.Proxy ESV1)
119−
126+#endif
120
127
121-------------------------------------------------------------------------------
128-------------------------------------------------------------------------------
122-- | EsScribeCfg that will use ElasticSearch V5
129-- | EsScribeCfg that will use ElasticSearch V5
@@ -240,7 +247,10 @@ splitTime t = asMins `divMod` 60
240
247
241-------------------------------------------------------------------------------
248-------------------------------------------------------------------------------
242data EsScribeSetupError = CouldNotCreateIndex !(Response ByteString)
249data EsScribeSetupError = CouldNotCreateIndex !(Response ByteString)
250+ | CouldNotUpdateIndexSettings !(Response ByteString)
243− | CouldNotCreateMapping !(Response ByteString) deriving (Typeable, Show)
251+ | CouldNotCreateMapping !(Response ByteString)
252+ | CouldNotPutTemplate !(Response ByteString)
253+ deriving (Typeable, Show)
244
254
245
255
246instance Exception EsScribeSetupError
256instance Exception EsScribeSetupError
@@ -265,26 +275,34 @@ mkEsScribe
265 -> IndexName v
275 -> IndexName v
266 -- ^ Treated as a prefix if index sharding is enabled
276 -- ^ Treated as a prefix if index sharding is enabled
267 -> MappingName v
277 -> MappingName v
268− -> Severity
278+ -> PermitFunc
269 -> Verbosity
279 -> Verbosity
270 -> IO Scribe
280 -> IO Scribe
271−mkEsScribe cfg@EsScribeCfg {..} env ix mapping sev verb = do
281+mkEsScribe cfg@EsScribeCfg {..} env ix mapping permit verb = do
272 q <- newTBMQueueIO $ unEsQueueSize essQueueSize
282 q <- newTBMQueueIO $ unEsQueueSize essQueueSize
273 endSig <- newEmptyMVar
283 endSig <- newEmptyMVar
274
284
275 runBH prx env $ do
285 runBH prx env $ do
276− chk <- indexExists prx ix
277− -- note that this doesn't update settings. That's not available
278− -- through the Bloodhound API yet
279− unless chk $ void $ do
280− r1 <- createIndex prx essIndexSettings ix
281− unless (statusIsSuccessful (responseStatus r1)) $
282− liftIO $ throwIO (CouldNotCreateIndex r1)
283− r2 <- if shardingEnabled
286+ if shardingEnabled
287+ then do
288+ -- create or update
284− then putTemplate prx tpl tplName
289+ res <- putTemplate prx tpl tplName
290+ unless (statusIsSuccessful (responseStatus res)) $
291+ liftIO $ EX.throwIO (CouldNotPutTemplate res)
292+ else do
293+ ixExists <- indexExists prx ix
294+ if ixExists
295+ then do
296+ res <- updateIndexSettings prx (toUpdatabaleIndexSettings prx essIndexSettings) ix
297+ unless (statusIsSuccessful (responseStatus res)) $
298+ liftIO $ EX.throwIO (CouldNotUpdateIndexSettings res)
299+ else do
300+ r1 <- createIndex prx essIndexSettings ix
301+ unless (statusIsSuccessful (responseStatus r1)) $
302+ liftIO $ EX.throwIO (CouldNotCreateIndex r1)
285− else putMapping prx ix mapping base
303+ r2 <- putMapping prx ix mapping base
286− unless (statusIsSuccessful (responseStatus r2)) $
304+ unless (statusIsSuccessful (responseStatus r2)) $
287− liftIO $ throwIO (CouldNotCreateMapping r2)
305+ liftIO $ EX.throwIO (CouldNotCreateMapping r2)
288
306
289 workers <- replicateM (unEsPoolSize essPoolSize) $ async $
307 workers <- replicateM (unEsPoolSize essPoolSize) $ async $
290 startWorker cfg env mapping q
308 startWorker cfg env mapping q
@@ -296,10 +314,10 @@ mkEsScribe cfg@EsScribeCfg {..} env ix mapping sev verb = do
296 putMVar endSig ()
314 putMVar endSig ()
297
315
298 let finalizer = putMVar endSig () >> takeMVar endSig
316 let finalizer = putMVar endSig () >> takeMVar endSig
299− return (Scribe (logger q) finalizer)
317+ return (Scribe (logger q) finalizer permit)
300 where
318 where
301 logger :: forall a. LogItem a => TBMQueue (IndexName v, Value) -> Item a -> IO ()
319 logger :: forall a. LogItem a => TBMQueue (IndexName v, Value) -> Item a -> IO ()
302− logger q i = when (_itemSeverity i >= sev) $
320+ logger q i =
303 void $ atomically $ tryWriteTBMQueue q (chooseIxn prx ix essIndexSharding i, itemJson' i)
321 void $ atomically $ tryWriteTBMQueue q (chooseIxn prx ix essIndexSharding i, itemJson' i)
304 prx :: Typeable.Proxy v
322 prx :: Typeable.Proxy v
305 prx = Typeable.Proxy
323 prx = Typeable.Proxy
@@ -320,24 +338,27 @@ mkEsScribe cfg@EsScribeCfg {..} env ix mapping sev verb = do
320baseMapping :: ESVersion v => proxy v -> MappingName v -> Value
338baseMapping :: ESVersion v => proxy v -> MappingName v -> Value
321baseMapping prx mn =
339baseMapping prx mn =
322 object [ fromMappingName prx mn .= object ["properties" .= object prs] ]
340 object [ fromMappingName prx mn .= object ["properties" .= object prs] ]
323− where prs = [ str "thread"
341+ where prs = [ unanalyzedString "thread"
324− , str "sev"
342+ , unanalyzedString "sev"
325− , str "pid"
343+ , unanalyzedString "pid"
326− , str "ns"
344+ -- ns is frequently fulltext searched
327− , str "msg"
345+ , analyzedString "ns"
346+ -- we want message to be fulltext searchable
347+ , analyzedString "msg"
328 , "loc" .= locType
348 , "loc" .= locType
329− , str "host"
349+ , unanalyzedString "host"
330− , str "env"
350+ , unanalyzedString "env"
331 , "at" .= dateType
351 , "at" .= dateType
332− , str "app"
352+ , unanalyzedString "app"
333 ]
353 ]
334− str k = k .= object ["type" .= String "string"]
354+ unanalyzedString k = k .= unanalyzedStringSpec prx
355+ analyzedString k = k .= analyzedStringSpec prx
335 locType = object ["properties" .= object locPairs]
356 locType = object ["properties" .= object locPairs]
336− locPairs = [ str "loc_pkg"
357+ locPairs = [ unanalyzedString "loc_pkg"
337− , str "loc_mod"
358+ , unanalyzedString "loc_mod"
338− , str "loc_ln"
359+ , unanalyzedString "loc_ln"
339− , str "loc_fn"
360+ , unanalyzedString "loc_fn"
340− , str "loc_col"
361+ , unanalyzedString "loc_col"
341 ]
362 ]
342 dateType = object [ "format" .= esDateFormat
363 dateType = object [ "format" .= esDateFormat
343 , "type" .= String "date"
364 , "type" .= String "date"
@@ -406,7 +427,7 @@ startWorker EsScribeCfg {..} env mapping q = go
406 popped <- atomically $ readTBMQueue q
427 popped <- atomically $ readTBMQueue q
407 case popped of
428 case popped of
408 Just (ixn, v) -> do
429 Just (ixn, v) -> do
409− sendLog ixn v `catchAny` eat
430+ sendLog ixn v `EX.catchAny` eat
410 go
431 go
411 Nothing -> return ()
432 Nothing -> return ()
412 prx :: Typeable.Proxy v
433 prx :: Typeable.Proxy v
@@ -419,8 +440,8 @@ startWorker EsScribeCfg {..} env mapping q = go
419 eat _ = return ()
440 eat _ = return ()
420 handler _ = Handler $ \e ->
441 handler _ = Handler $ \e ->
421 case fromException e of
442 case fromException e of
422− Just (_ :: AsyncException) -> return False
443+ Just (_ :: EX.SomeAsyncException) -> return False
423− _ -> return True
444+ _ -> return True
424
445
425
446
426-------------------------------------------------------------------------------
447-------------------------------------------------------------------------------
@@ -433,6 +454,7 @@ class ESVersion v where
433 type BHEnv v
454 type BHEnv v
434 type IndexSettings v
455 type IndexSettings v
435 defaultIndexSettings :: proxy v -> IndexSettings v
456 defaultIndexSettings :: proxy v -> IndexSettings v
457+ type UpdatableIndexSetting v
436 type IndexName v
458 type IndexName v
437 toIndexName :: proxy v -> Text -> IndexName v
459 toIndexName :: proxy v -> Text -> IndexName v
438 fromIndexName :: proxy v -> IndexName v -> Text
460 fromIndexName :: proxy v -> IndexName v -> Text
@@ -451,22 +473,35 @@ class ESVersion v where
451 type IndexDocumentSettings v
473 type IndexDocumentSettings v
452 defaultIndexDocumentSettings :: proxy v -> IndexDocumentSettings v
474 defaultIndexDocumentSettings :: proxy v -> IndexDocumentSettings v
453
475
476+ toUpdatabaleIndexSettings :: proxy v -> IndexSettings v -> NonEmpty (UpdatableIndexSetting v)
477+
454 -- Operations
478 -- Operations
455 -- We're deciding on IO here, but it isn't necessary
479 -- We're deciding on IO here, but it isn't necessary
456 indexExists :: proxy v -> IndexName v -> BH v IO Bool
480 indexExists :: proxy v -> IndexName v -> BH v IO Bool
457 indexDocument :: ToJSON doc => proxy v -> IndexName v -> MappingName v -> IndexDocumentSettings v -> doc -> DocId v -> BH v IO (Response ByteString)
481 indexDocument :: ToJSON doc => proxy v -> IndexName v -> MappingName v -> IndexDocumentSettings v -> doc -> DocId v -> BH v IO (Response ByteString)
458 createIndex :: proxy v -> IndexSettings v -> IndexName v -> BH v IO (Response ByteString)
482 createIndex :: proxy v -> IndexSettings v -> IndexName v -> BH v IO (Response ByteString)
483+ updateIndexSettings :: proxy v -> NonEmpty (UpdatableIndexSetting v) -> IndexName v -> BH v IO (Response ByteString)
459 putTemplate :: proxy v -> IndexTemplate v -> TemplateName v -> BH v IO (Response ByteString)
484 putTemplate :: proxy v -> IndexTemplate v -> TemplateName v -> BH v IO (Response ByteString)
460 putMapping :: (ToJSON a) => proxy v -> IndexName v -> MappingName v -> a -> BH v IO (Response ByteString)
485 putMapping :: (ToJSON a) => proxy v -> IndexName v -> MappingName v -> a -> BH v IO (Response ByteString)
461
486
487+ -- In ES 5 and beyond, "string" was deprecated in favor of text for
488+ -- fulltext and keyword for unanalyzed tokens
489+ unanalyzedStringSpec :: proxy v -> Value
490+ analyzedStringSpec :: proxy v -> Value
491+
492+
462
493
494+#if !MIN_VERSION_bloodhound(0,17,0)
463data ESV1 = ESV1
495data ESV1 = ESV1
496+{-# DEPRECATED ESV1 "ESV1 is deprecated and removed in bloodhound >= 0.17.0" #-}
497+
464
498
465instance ESVersion ESV1 where
499instance ESVersion ESV1 where
466 type BHEnv ESV1 = V1.BHEnv
500 type BHEnv ESV1 = V1.BHEnv
467 type IndexSettings ESV1 = V1.IndexSettings
501 type IndexSettings ESV1 = V1.IndexSettings
468 defaultIndexSettings _ = V1.defaultIndexSettings
502 defaultIndexSettings _ = V1.defaultIndexSettings
469 type IndexName ESV1 = V1.IndexName
503 type IndexName ESV1 = V1.IndexName
504+ type UpdatableIndexSetting ESV1 = V1.UpdatableIndexSetting
470 toIndexName _ = V1.IndexName
505 toIndexName _ = V1.IndexName
471 fromIndexName _ (V1.IndexName x) = x
506 fromIndexName _ (V1.IndexName x) = x
472 type MappingName ESV1 = V1.MappingName
507 type MappingName ESV1 = V1.MappingName
@@ -482,12 +517,24 @@ instance ESVersion ESV1 where
482 type IndexTemplate ESV1 = V1.IndexTemplate
517 type IndexTemplate ESV1 = V1.IndexTemplate
483 toIndexTemplate _ = V1.IndexTemplate
518 toIndexTemplate _ = V1.IndexTemplate
484 type IndexDocumentSettings ESV1 = V1.IndexDocumentSettings
519 type IndexDocumentSettings ESV1 = V1.IndexDocumentSettings
520+ toUpdatabaleIndexSettings _ s =
521+ (V1.NumberOfReplicas (V1.indexReplicas s)) :| []
485 defaultIndexDocumentSettings _ = V1.defaultIndexDocumentSettings
522 defaultIndexDocumentSettings _ = V1.defaultIndexDocumentSettings
486 indexExists _ = V1.indexExists
523 indexExists _ = V1.indexExists
487 indexDocument _ = V1.indexDocument
524 indexDocument _ = V1.indexDocument
488 createIndex _ = V1.createIndex
525 createIndex _ = V1.createIndex
526+ updateIndexSettings _ = V1.updateIndexSettings
489 putTemplate _ = V1.putTemplate
527 putTemplate _ = V1.putTemplate
490 putMapping _ = V1.putMapping
528 putMapping _ = V1.putMapping
529+ unanalyzedStringSpec _ = object
530+ [ "type" .= String "string"
531+ , "index" .= String "not_analyzed"
532+ ]
533+ analyzedStringSpec _ = object
534+ [ "type" .= String "string"
535+ , "index" .= String "analyzed"
536+ ]
537+#endif
491
538
492
539
493data ESV5 = ESV5
540data ESV5 = ESV5
@@ -495,6 +542,7 @@ data ESV5 = ESV5
495instance ESVersion ESV5 where
542instance ESVersion ESV5 where
496 type BHEnv ESV5 = V5.BHEnv
543 type BHEnv ESV5 = V5.BHEnv
497 type IndexSettings ESV5 = V5.IndexSettings
544 type IndexSettings ESV5 = V5.IndexSettings
545+ type UpdatableIndexSetting ESV5 = V5.UpdatableIndexSetting
498 defaultIndexSettings _ = V5.defaultIndexSettings
546 defaultIndexSettings _ = V5.defaultIndexSettings
499 type IndexName ESV5 = V5.IndexName
547 type IndexName ESV5 = V5.IndexName
500 toIndexName _ = V5.IndexName
548 toIndexName _ = V5.IndexName
@@ -512,9 +560,18 @@ instance ESVersion ESV5 where
512 type IndexTemplate ESV5 = V5.IndexTemplate
560 type IndexTemplate ESV5 = V5.IndexTemplate
513 toIndexTemplate _ = V5.IndexTemplate
561 toIndexTemplate _ = V5.IndexTemplate
514 type IndexDocumentSettings ESV5 = V5.IndexDocumentSettings
562 type IndexDocumentSettings ESV5 = V5.IndexDocumentSettings
563+ toUpdatabaleIndexSettings _ s =
564+ (V5.NumberOfReplicas (V5.indexReplicas s)) :| []
515 defaultIndexDocumentSettings _ = V5.defaultIndexDocumentSettings
565 defaultIndexDocumentSettings _ = V5.defaultIndexDocumentSettings
516 indexExists _ = V5.indexExists
566 indexExists _ = V5.indexExists
517 indexDocument _ = V5.indexDocument
567 indexDocument _ = V5.indexDocument
518 createIndex _ = V5.createIndex
568 createIndex _ = V5.createIndex
569+ updateIndexSettings _ = V5.updateIndexSettings
519 putTemplate _ = V5.putTemplate
570 putTemplate _ = V5.putTemplate
520 putMapping _ = V5.putMapping
571 putMapping _ = V5.putMapping
572+ unanalyzedStringSpec _ = object
573+ [ "type" .= String "keyword"
574+ ]
575+ analyzedStringSpec _ = object
576+ [ "type" .= String "text"
577+ ]
katip-elasticsearch/test/Main.hs+35−10
@@ -17,6 +17,7 @@ module Main
17-------------------------------------------------------------------------------
17-------------------------------------------------------------------------------
18import Control.Applicative as A
18import Control.Applicative as A
19import Control.Concurrent.STM
19import Control.Concurrent.STM
20+import Control.Exception
20import Control.Lens hiding (mapping, (.=))
21import Control.Lens hiding (mapping, (.=))
21import Control.Monad
22import Control.Monad
22import Control.Monad.IO.Class
23import Control.Monad.IO.Class
@@ -25,7 +26,7 @@ import Data.Aeson.Lens
25import Data.Aeson.Types
26import Data.Aeson.Types
26import Data.ByteString.Lazy (ByteString)
27import Data.ByteString.Lazy (ByteString)
27import qualified Data.HashMap.Strict as HM
28import qualified Data.HashMap.Strict as HM
28−import Data.Monoid
29+import Data.Monoid as M
29import Data.Scientific
30import Data.Scientific
30import Data.Tagged
31import Data.Tagged
31import Data.Text (Text)
32import Data.Text (Text)
@@ -33,8 +34,12 @@ import Data.Time
33import Data.Time.Calendar.WeekDate
34import Data.Time.Calendar.WeekDate
34import Data.Typeable as Typeable
35import Data.Typeable as Typeable
35import qualified Data.Vector as V
36import qualified Data.Vector as V
37+#if MIN_VERSION_bloodhound(0,17,0)
38+import qualified Database.Bloodhound as V5
39+#else
36import qualified Database.V1.Bloodhound as V1
40import qualified Database.V1.Bloodhound as V1
37import qualified Database.V5.Bloodhound as V5
41import qualified Database.V5.Bloodhound as V5
42+#endif
38import Network.HTTP.Client
43import Network.HTTP.Client
39import Network.HTTP.Types.Status
44import Network.HTTP.Types.Status
40import Test.QuickCheck.Instances ()
45import Test.QuickCheck.Instances ()
@@ -53,8 +58,10 @@ main :: IO ()
53main = defaultMainWithIngredients ings $ askOption $ \vers -> testGroup "katip-elasticsearch"
58main = defaultMainWithIngredients ings $ askOption $ \vers -> testGroup "katip-elasticsearch"
54 [
59 [
55 case vers of
60 case vers of
61+#if !MIN_VERSION_bloodhound(0,17,0)
56− TestV1 -> esTests (Typeable.Proxy :: Typeable.Proxy ESV1)
62+ TestV1 -> esTests (Typeable.Proxy :: Typeable.Proxy ESV1)
63+#endif
57− TestV5 -> esTests (Typeable.Proxy :: Typeable.Proxy ESV5)
64+ TestV5 -> esTests (Typeable.Proxy :: Typeable.Proxy ESV5)
58 , typeAnnotatedTests
65 , typeAnnotatedTests
59 , roundToSundayTests
66 , roundToSundayTests
60 ]
67 ]
@@ -63,18 +70,26 @@ main = defaultMainWithIngredients ings $ askOption $ \vers -> testGroup "katip-e
63
70
64
71
65-------------------------------------------------------------------------------
72-------------------------------------------------------------------------------
66−data TestWithESVersion = TestV1
73+data TestWithESVersion = TestV5
67− | TestV5
74+#if !MIN_VERSION_bloodhound(0,17,0)
75+ | TestV1
76+#endif
68− deriving (Typeable)
77+ deriving (Typeable)
69
78
70
79
71instance IsOption TestWithESVersion where
80instance IsOption TestWithESVersion where
72− defaultValue = TestV1
81+ defaultValue = TestV5
82+#if !MIN_VERSION_bloodhound(0,17,0)
73 parseValue "1" = Just TestV1
83 parseValue "1" = Just TestV1
84+#endif
74 parseValue "5" = Just TestV5
85 parseValue "5" = Just TestV5
75 parseValue _ = Nothing
86 parseValue _ = Nothing
76 optionName = Tagged "es-version"
87 optionName = Tagged "es-version"
88+#if MIN_VERSION_bloodhound(0,17,0)
77− optionHelp = Tagged "Version of ES to test against, either 1 or 5, defaulting to 1."
89+ optionHelp = Tagged "Version of ES to test against, either 1 or 5, defaulting to 5."
90+#else
91+ optionHelp = Tagged "Version of ES to test against, only v5 is supported, but should work for higher versions."
92+#endif
78
93
79
94
80class ESVersion v => TestESVersion v where
95class ESVersion v => TestESVersion v where
@@ -100,6 +115,7 @@ class ESVersion v => TestESVersion v where
100 searchByIndex :: proxy v -> IndexName v -> Search v -> BH v IO (Response ByteString)
115 searchByIndex :: proxy v -> IndexName v -> Search v -> BH v IO (Response ByteString)
101
116
102
117
118+#if !MIN_VERSION_bloodhound(0,17,0)
103instance TestESVersion ESV1 where
119instance TestESVersion ESV1 where
104 type Server ESV1 = V1.Server
120 type Server ESV1 = V1.Server
105 toServer _ = V1.Server
121 toServer _ = V1.Server
@@ -121,6 +137,7 @@ instance TestESVersion ESV1 where
121 refreshIndex _ = V1.refreshIndex
137 refreshIndex _ = V1.refreshIndex
122 withBH _ = V1.withBH
138 withBH _ = V1.withBH
123 searchByIndex _ = V1.searchByIndex
139 searchByIndex _ = V1.searchByIndex
140+#endif
124
141
125
142
126instance TestESVersion ESV5 where
143instance TestESVersion ESV5 where
@@ -160,7 +177,7 @@ setupSearch
160setupSearch prx modScribeCfg = do
177setupSearch prx modScribeCfg = do
161 bh prx (dropESSchema prx)
178 bh prx (dropESSchema prx)
162 mgr <- newManager defaultManagerSettings
179 mgr <- newManager defaultManagerSettings
163− mkEsScribe cfg (mkBHEnv prx (svr prx) mgr) (ixn prx) (mn prx) DebugS V3
180+ mkEsScribe cfg (mkBHEnv prx (svr prx) mgr) (ixn prx) (mn prx) (permitItem DebugS) V3
164 where
181 where
165 cfg :: EsScribeCfg v
182 cfg :: EsScribeCfg v
166 cfg = modScribeCfg $
183 cfg = modScribeCfg $
@@ -229,7 +246,7 @@ esTests
229esTests prx = testGroup "elasticsearch scribe"
246esTests prx = testGroup "elasticsearch scribe"
230 [
247 [
231 withSearch' (\c -> c { essIndexSharding = NoIndexSharding}) prx $ \setup -> testCase "it flushes to elasticsearch" $ withTestLogging prx setup $ \done -> do
248 withSearch' (\c -> c { essIndexSharding = NoIndexSharding}) prx $ \setup -> testCase "it flushes to elasticsearch" $ withTestLogging prx setup $ \done -> do
232− $(logT) (ExampleCtx True) mempty InfoS "A test message"
249+ $(logT) (ExampleCtx True) M.mempty InfoS "A test message"
233 liftIO $ do
250 liftIO $ do
234 void done
251 void done
235 logs <- getLogs prx
252 logs <- getLogs prx
@@ -237,6 +254,14 @@ esTests prx = testGroup "elasticsearch scribe"
237 let l = head logs
254 let l = head logs
238 l ^? key "_source" . key "msg" . _String @?= Just "A test message"
255 l ^? key "_source" . key "msg" . _String @?= Just "A test message"
239 l ^? key "_source" . key "data" . key "whatever::b" . _Bool @?= Just True
256 l ^? key "_source" . key "data" . key "whatever::b" . _Bool @?= Just True
257+ , testCase "can set up twice with no sharding" $ do
258+ let setup = bracket_ (setupSearch prx (\c -> c { essIndexSharding = NoIndexSharding })) (teardownSearch prx)
259+ setup $ setup $
260+ return ()
261+ , testCase "can set up twice with sharding" $ do
262+ let setup = bracket_ (setupSearch prx (\c -> c { essIndexSharding = DailyIndexSharding })) (teardownSearch prx)
263+ setup $ setup $
264+ return ()
240 , withSearch prx $ \setup -> testCase "date-based index sharding" $ do
265 , withSearch prx $ \setup -> testCase "date-based index sharding" $ do
241 let t1 = mkTime 2016 1 2 3 4 5
266 let t1 = mkTime 2016 1 2 3 4 5
242 fakeClock <- newTVarIO t1
267 fakeClock <- newTVarIO t1
katip-logzio/LICENSEnew file+30
@@ -0,0 +1,30 @@
1+Copyright (c) 2018, Soostone Inc
2+
3+All rights reserved.
4+
5+Redistribution and use in source and binary forms, with or without
6+modification, are permitted provided that the following conditions are met:
7+
8+ * Redistributions of source code must retain the above copyright
9+ notice, this list of conditions and the following disclaimer.
10+
11+ * Redistributions in binary form must reproduce the above
12+ copyright notice, this list of conditions and the following
13+ disclaimer in the documentation and/or other materials provided
14+ with the distribution.
15+
16+ * Neither the name of Ozgun Ataman nor the names of other
17+ contributors may be used to endorse or promote products derived
18+ from this software without specific prior written permission.
19+
20+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
21+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
22+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
23+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
24+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
25+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
26+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
27+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
28+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
29+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
30+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
katip-logzio/README.mdnew file+4
@@ -0,0 +1,4 @@
1+# Katip Logz.IO [![Build Status](https://travis-ci.org/Soostone/katip.svg?branch=master)](https://travis-ci.org/Soostone/katip)
2+
3+katip-logzio is a scribe for the Katip logging framework that sends
4+structured logs to the logz.io centralized logging platform.
katip-logzio/changelog.mdnew file+9
@@ -0,0 +1,9 @@
1+0.1.1.0
2+=======
3+* Fixed bug where on scribe close, any items left in the buffer that
4+ hadn't timed out or been flushed already would be dropped rather
5+ than flushed. All users are encouraged to upgrade.
6+
7+0.1.0.0
8+==========
9+* Initial release
katip-logzio/katip-logzio.cabalnew file+94
@@ -0,0 +1,94 @@
1+name: katip-logzio
2+version: 0.1.1.0
3+synopsis: Logz.IO scribe for the Katip logging framework
4+description: See README.md for more details.
5+license: BSD3
6+license-file: LICENSE
7+author: Michael Xavier
8+maintainer: michael.xavier@soostone.com
9+copyright: Soostone Inc, 2018
10+category: Data, Text, Logging
11+homepage: https://github.com/Soostone/katip
12+bug-reports: https://github.com/Soostone/katip/issues
13+build-type: Simple
14+cabal-version: >=1.10
15+extra-source-files:
16+ README.md
17+ changelog.md
18+
19+tested-with: GHC == 8.4.3
20+
21+source-repository head
22+ type: git
23+ location: https://github.com/Soostone/katip.git
24+
25+flag lib-Werror
26+ default: False
27+ manual: True
28+
29+
30+library
31+ exposed-modules:
32+ Katip.Scribes.LogzIO.HTTPS
33+ build-depends:
34+ base >=4.8.0.0 && <5
35+ , katip >= 0.8.0.0
36+ , stm >= 2.1.1
37+ , async
38+ , time >= 1.8.0.2
39+ , aeson >= 0.10.0.0
40+ , unix
41+ , unordered-containers
42+ , http-client >= 0.1.0
43+ , http-client-tls >= 0.3.4
44+ , uri-bytestring
45+ , errors >= 2.0.0
46+ , safe-exceptions
47+ , retry
48+ , http-types
49+ , text
50+ , bytestring
51+ , scientific
52+ , stm-chans >= 2.0.0
53+ hs-source-dirs: src
54+ default-language: Haskell2010
55+ hs-source-dirs: src
56+ ghc-options: -Wall -Wall -Wnoncanonical-monad-instances
57+ if flag(lib-Werror)
58+ ghc-options: -Werror
59+
60+
61+test-suite test
62+ type: exitcode-stdio-1.0
63+ main-is: Main.hs
64+ hs-source-dirs: test
65+ default-language: Haskell2010
66+ other-modules:
67+ Katip.Tests.Scribes.LogzIO.HTTPS
68+ build-depends: base
69+ , katip
70+ , katip-logzio
71+ , tasty
72+ , tasty-hedgehog
73+ , tasty-hunit
74+ , hedgehog
75+ , vector
76+ , unordered-containers
77+ , text
78+ , bytestring
79+ , time
80+ , template-haskell
81+ , aeson
82+ , scientific
83+ , unix
84+ , hostname
85+ , scotty
86+ , stm
87+ , async
88+ , safe-exceptions
89+ , uri-bytestring
90+ , warp
91+ , http-types
92+ ghc-options: -Wall -threaded -rtsopts -with-rtsopts=-N -Wnoncanonical-monad-instances
93+ if flag(lib-Werror)
94+ ghc-options: -Werror
katip-logzio/src/Katip/Scribes/LogzIO/HTTPS.hsnew file+639
@@ -0,0 +1,639 @@
1+{-# LANGUAGE BangPatterns #-}
2+{-# LANGUAGE GADTs #-}
3+{-# LANGUAGE GeneralizedNewtypeDeriving #-}
4+{-# LANGUAGE OverloadedStrings #-}
5+{-# LANGUAGE RankNTypes #-}
6+{-# LANGUAGE TypeApplications #-}
7+-- | This is a log scribe that writes logs to logz.io's bulk
8+-- <https://app.logz.io/#/dashboard/data-sources/Bulk-HTTPS HTTPS
9+-- API>.
10+module Katip.Scribes.LogzIO.HTTPS
11+ ( -- * Scribe construction
12+ mkLogzIOScribe
13+ -- * Types
14+ , BulkAPIError(..)
15+ , LogzIOScribeConfiguration(..)
16+ , Scheme(..)
17+ , APIToken(..)
18+ , LoggingError(..)
19+ -- ** Presets for configuration
20+ , usRegionHost
21+ , euRegionHost
22+ , httpsPort
23+ , httpPort
24+ , defaultRetryPolicy
25+ , defaultLogzIOScribeConfiguration
26+ -- * Internal API exported for testing
27+ , renderLineTruncated
28+ , renderLineTruncated'
29+ , maxPayloadBytes
30+ , maxLogLineLength
31+ , BulkBuffer (..)
32+ , LogAction (..)
33+ , bufferItem
34+ , bufferItem'
35+ , forceFlush
36+ , Bytes (..)
37+ ) where
38+
39+
40+-------------------------------------------------------------------------------
41+import Control.Applicative
42+import qualified Control.Concurrent.Async as Async
43+import qualified Control.Concurrent.STM as STM
44+import qualified Control.Concurrent.STM.TBMQueue as STM
45+import qualified Control.Error as E
46+import qualified Control.Exception.Safe as EX
47+import Control.Monad
48+import qualified Control.Retry as Retry
49+import qualified Data.Aeson as A
50+import qualified Data.ByteString.Builder as BB
51+import qualified Data.ByteString.Lazy as LBS
52+import qualified Data.ByteString.Lazy.Char8 as LBS8
53+import qualified Data.HashMap.Strict as HM
54+import Data.Int
55+import qualified Data.Scientific as Scientific
56+import Data.Semigroup as Semigroup
57+import Data.String (IsString)
58+import qualified Data.Text as T
59+import qualified Data.Text.Encoding as TE
60+import qualified Data.Text.Lazy as TL
61+import qualified Data.Text.Lazy.Builder as TB
62+import qualified Data.Time as Time
63+import qualified Katip as K
64+import Katip.Core (LocJs (..))
65+import qualified Network.HTTP.Client as HTTP
66+import qualified Network.HTTP.Client.TLS as HTTPS
67+import qualified Network.HTTP.Types as HTypes
68+import qualified System.Posix.Types as POSIX
69+import qualified URI.ByteString as URIBS
70+-------------------------------------------------------------------------------
71+
72+
73+
74+-- | This is returned when the bulk import was a partial success
75+data BulkAPIError = BulkAPIError
76+ { bulkAPIError_malformedLines :: Int
77+ -- ^ The number of log lines which are not well-formed JSON. This
78+ -- indicates a __library bug__. Please file an issue on GitHub.
79+ , bulkAPIError_successfulLines :: Int
80+ -- ^ The number of log lines received successfully.
81+ , bulkAPIError_oversizedLines :: Int
82+ -- ^ The number of log lines which exceed the line length
83+ -- limit. katip-logzio makes a best effort to truncate logs that
84+ -- exceed the size limit. This probably indicates a __library bug__
85+ -- and should be reported as an issue on GitHub.
86+ , bulkAPIError_emptyLogLines :: Int
87+ -- ^ The number of log lines which were empty. There isnt' really a
88+ -- concept of a truly empty log line in katip, so this most likely
89+ -- indicates a __library bug__ and should be reported as an issue on
90+ -- GitHub.
91+ } deriving (Show, Eq)
92+
93+
94+instance A.FromJSON BulkAPIError where
95+ parseJSON = A.withObject "BulkAPIError" $ \o -> do
96+ malformedLines <- o A..: "malformedLines"
97+ successfulLines <- o A..: "successfulLines"
98+ oversizedLines <- o A..: "oversizedLines"
99+ emptyLogLines <- o A..: "emptyLogLines"
100+ pure $ BulkAPIError
101+ { bulkAPIError_malformedLines = malformedLines
102+ , bulkAPIError_successfulLines = successfulLines
103+ , bulkAPIError_oversizedLines = oversizedLines
104+ , bulkAPIError_emptyLogLines = emptyLogLines
105+ }
106+
107+
108+-------------------------------------------------------------------------------
109+data LogzIOScribeConfiguration = LogzIOScribeConfiguration
110+ { logzIOScribeConfiguration_bufferItems :: Int
111+ -- ^ Will flush the log buffer if this many items is in the buffer
112+ -- __or__ 'logzIOScribeConfiguration_bufferTimeout' is reached,
113+ -- whichever is first
114+ , logzIOScribeConfiguration_bufferTimeout :: Time.NominalDiffTime
115+ -- ^ Will flush the buffer if it has been this long since the last
116+ -- flush __or__ 'logzIOScribeConfiguration_bufferItems' items are
117+ -- accumulated, whichever is first. NominalDiffTime has a Num
118+ -- instance, so you can use a literal to specify seconds, e.g. 30 =
119+ -- 30s.
120+ , logzIOScribeConfiguration_scheme :: Scheme
121+ , logzIOScribeConfiguration_host :: URIBS.Host
122+ , logzIOScribeConfiguration_port :: URIBS.Port
123+ , logzIOScribeConfiguration_token :: APIToken
124+ , logzIOScribeConfiguration_retry :: Retry.RetryPolicyM IO
125+ -- ^ How should exceptions during writes be retried?
126+ , logzIOScribeConfiguration_onError :: LoggingError -> IO ()
127+ }
128+
129+
130+-- | A default configuration:
131+--
132+-- * 100 item buffering
133+--
134+-- * 30 second buffer timeout
135+--
136+-- * US, HTTPS logging endpoint (listener.logz.io:8071)
137+--
138+-- * 'defaultRetryPolicy' of 25ms exponential backoff, 5 retries
139+--
140+-- * Ignore logging errors
141+defaultLogzIOScribeConfiguration :: APIToken -> LogzIOScribeConfiguration
142+defaultLogzIOScribeConfiguration token = LogzIOScribeConfiguration
143+ { logzIOScribeConfiguration_bufferItems = 100
144+ , logzIOScribeConfiguration_bufferTimeout = 30
145+ , logzIOScribeConfiguration_scheme = HTTPS
146+ , logzIOScribeConfiguration_host = usRegionHost
147+ , logzIOScribeConfiguration_port = httpsPort
148+ , logzIOScribeConfiguration_token = token
149+ , logzIOScribeConfiguration_retry = defaultRetryPolicy
150+ , logzIOScribeConfiguration_onError = const (pure ())
151+ }
152+
153+
154+-------------------------------------------------------------------------------
155+-- | You can retrieve your account or sub-account's API token on the
156+-- <https://app.logz.io/#/dashboard/settings/manage-accounts manage
157+-- accounts page>. Note that APIToken has an IsString instance,
158+-- meaning that you can use a string literal with OverloadedStrings
159+-- enabled.
160+newtype APIToken = APIToken
161+ { apiToken :: T.Text
162+ } deriving (Show, Eq, IsString)
163+
164+
165+-- | This particular bulk API only supports HTTP or HTTPS. HTTPS is
166+-- strongly recommended for security reasons.
167+data Scheme
168+ = HTTPS
169+ -- ^ HTTPs should always be used except for local testing.
170+ | HTTP
171+ deriving (Show, Eq)
172+
173+
174+-- | See
175+-- <https://support.logz.io/hc/en-us/articles/210206365-What-IP-addresses-should-I-open-in-my-firewall-to-ship-logs-to-Logz-io-
176+-- this> for a list of listeners. This is the US region host,
177+-- listener.logz.io
178+usRegionHost :: URIBS.Host
179+usRegionHost = URIBS.Host "listener.logz.io"
180+
181+
182+-- | See
183+-- <https://support.logz.io/hc/en-us/articles/210206365-What-IP-addresses-should-I-open-in-my-firewall-to-ship-logs-to-Logz-io-
184+-- this> for a list of listeners. This is the EU region host,
185+-- listener.logz.io
186+euRegionHost :: URIBS.Host
187+euRegionHost = URIBS.Host "listener-eu.logz.io"
188+
189+
190+-- | Logz.io uses port 8071 for HTTPS
191+httpsPort :: URIBS.Port
192+httpsPort = URIBS.Port 8071
193+
194+
195+-- | Logz.io uses port 8070 for HTTP
196+httpPort :: URIBS.Port
197+httpPort = URIBS.Port 8070
198+
199+
200+-- | A reasonable retry policy: exponential backoff with 25ms base
201+-- delay up to 5 retries, for a total cumulative delay of 775ms.
202+defaultRetryPolicy :: (Monad m) => Retry.RetryPolicyM m
203+defaultRetryPolicy = Retry.exponentialBackoff 25000 `mappend` Retry.limitRetries 5
204+
205+-------------------------------------------------------------------------------
206+mkLogzIOScribe
207+ :: LogzIOScribeConfiguration
208+ -> K.PermitFunc
209+ -> K.Verbosity
210+ -> IO K.Scribe
211+mkLogzIOScribe config permitItem verbosity = do
212+ -- This size is actually somewhat arbitrary. We're just making sure
213+ -- it isn't infinite so we don't consume the whole backlog right
214+ -- away and take up lots of memory.. Katip above us does bounded
215+ -- buffering and load-shedding when there's too much of a
216+ -- backlog. Our writes are blocking to apply backpressure up to
217+ -- katip. This value just holds onto values temporarily while we
218+ -- concatenate them into a buffer.
219+ ingestionQueue <- STM.newTBMQueueIO (itemBufferSize * 10)
220+
221+ let newTimer = STM.registerDelay itemBufferTimeoutMicros
222+ timerRef <- STM.newTVarIO =<< newTimer
223+
224+ -- Set up a connection manager for requests
225+ mgr <- case scheme of
226+ HTTPS -> HTTPS.newTlsManager
227+ HTTP -> HTTP.newManager HTTP.defaultManagerSettings
228+
229+ -- An STM transaction that will return true when writes are stopped
230+ -- and backlog is emptied.
231+ let workExhausted :: STM.STM Bool
232+ workExhausted = (&&)
233+ <$> STM.isClosedTBMQueue ingestionQueue
234+ <*> STM.isEmptyTBMQueue ingestionQueue
235+ -- Block until there's no more work
236+ let waitWorkExhausted :: STM.STM ()
237+ waitWorkExhausted = STM.check =<< workExhausted
238+ let pop :: STM.STM (Tick AnyLogItem)
239+ pop = maybe WorkExhausted NewItem <$> STM.readTBMQueue ingestionQueue
240+
241+ let timeExpired :: STM.STM (Tick a)
242+ timeExpired = do
243+ isExpired <- STM.readTVar =<< STM.readTVar timerRef
244+ STM.check isExpired
245+ pure TimeExpired
246+
247+ -- Circular transaction that checks for completion of work, time
248+ -- expiration, or new events.
249+ let nextTick :: STM.STM (Tick AnyLogItem)
250+ nextTick =
251+ timeExpired <|>
252+ pop <|>
253+ (WorkExhausted <$ waitWorkExhausted)
254+
255+ -- Blocking push. This applies backpressure upstream to katip, which does its own buffering
256+ let push :: AnyLogItem -> STM.STM ()
257+ push = void . STM.writeTBMQueue ingestionQueue
258+
259+ let sealQueue = STM.atomically (STM.closeTBMQueue ingestionQueue)
260+
261+ -- Replace the timer with a new one
262+ let resetTimer = STM.atomically . STM.writeTVar timerRef =<< newTimer
263+
264+ -- Send the buffer and then reset the timer
265+ let flush curBuffer = do
266+ res <- flushBuffer config mgr curBuffer
267+ case res of
268+ Left e -> onErrorSafe e
269+ Right () -> pure ()
270+ resetTimer
271+
272+ let flushLoop :: BulkBuffer -> IO ()
273+ flushLoop curBuffer = do
274+ tick <- STM.atomically nextTick
275+ case tick of
276+ WorkExhausted -> do
277+ flush curBuffer -- flush what you've got
278+ pure () -- stop looping
279+ TimeExpired -> do
280+ flush curBuffer
281+ flushLoop mempty
282+ NewItem (AnyLogItem item) -> do
283+ case bufferItem (logzIOScribeConfiguration_bufferItems config) verbosity item curBuffer of
284+ Buffered newBuffer -> flushLoop newBuffer
285+ FlushNow flushThis newBuffer -> do
286+ flush flushThis
287+ flushLoop newBuffer
288+
289+ flushThread <- Async.async (flushLoop mempty)
290+
291+ let close = do
292+ sealQueue
293+ _ <- Async.waitCatch flushThread
294+ pure ()
295+
296+ pure $ K.Scribe
297+ { K.liPush = STM.atomically . push . AnyLogItem
298+ , K.scribeFinalizer = close
299+ , K.scribePermitItem = permitItem
300+ }
301+ where
302+ itemBufferSize = logzIOScribeConfiguration_bufferItems config
303+ itemBufferTimeoutMicros = ndtToMicros (logzIOScribeConfiguration_bufferTimeout config)
304+ scheme = logzIOScribeConfiguration_scheme config
305+ onErrorSafe ex = do
306+ _ <- EX.tryAny (logzIOScribeConfiguration_onError config ex)
307+ pure ()
308+
309+
310+data AnyLogItem where
311+ AnyLogItem :: K.LogItem a => K.Item a -> AnyLogItem
312+
313+
314+-------------------------------------------------------------------------------
315+data Tick a =
316+ TimeExpired
317+ | NewItem !a
318+ | WorkExhausted
319+
320+
321+-------------------------------------------------------------------------------
322+-- Match the native resolution of NominalDiffTime, which is excessive
323+-- for our needs
324+ndtPicos :: Time.NominalDiffTime -> Int64
325+ndtPicos = round . (* picos)
326+ where
327+ picos :: Time.NominalDiffTime
328+ picos = 10 ^ (9 :: Int)
329+
330+
331+-------------------------------------------------------------------------------
332+ndtToMicros :: Time.NominalDiffTime -> Int
333+ndtToMicros t = round (((fromIntegral (ndtPicos t)) :: Double) / picosInMicro)
334+ where
335+ picosInMicro = 10 ^ (3 :: Int)
336+
337+
338+-------------------------------------------------------------------------------
339+data LoggingError =
340+ URIError HTTP.HttpException
341+ -- ^ The URI generated was invalid. Check your configuration
342+ | RequestError HTTP.HttpException
343+ -- ^ We encountered an exception while sending the batch request
344+ | PartialFailure BulkAPIError
345+ -- ^ Some or all of the request was rejected. Check the logz.io UI
346+ -- for indexing errors.
347+ | BadToken
348+ -- ^ Your API token was rejected.
349+ | UnknownFailureResponse HTypes.Status LBS.ByteString
350+ -- ^ An error returned, but it could not be decoded into a
351+ -- 'BulkAPIError'. This may indicate a __library bug__, which should
352+ -- be reported to the issue tracker.
353+ deriving (Show)
354+
355+-------------------------------------------------------------------------------
356+flushBuffer
357+ :: LogzIOScribeConfiguration
358+ -> HTTP.Manager
359+ -> BulkBuffer
360+ -> IO (Either LoggingError ())
361+flushBuffer config mgr bulkBuffer
362+ | bulkBuffer_itemCount bulkBuffer <= 0 = pure (Right ())
363+ | otherwise = do
364+ E.runExceptT $ do
365+ req <- E.fmapLT URIError (E.ExceptT (EX.try (configureRequest <$> HTTP.parseRequest uriStr)))
366+ resp <- E.fmapLT RequestError $ E.ExceptT $ EX.try $
367+ Retry.recovering retryPolicy [\_stat -> EX.Handler handleHttpException] $ \_stat ->
368+ HTTP.httpLbs req mgr
369+ let respLBS = HTTP.responseBody resp
370+ let respStatus = HTTP.responseStatus resp
371+ if HTypes.statusIsSuccessful respStatus
372+ then pure ()
373+ else case A.decode @BulkAPIError respLBS of
374+ Nothing
375+ | HTypes.statusCode respStatus == 401 -> E.throwE BadToken
376+ | otherwise -> E.throwE (UnknownFailureResponse respStatus respLBS)
377+ Just bulkError -> E.throwE (PartialFailure bulkError)
378+ where
379+ configureRequest req = req
380+ { HTTP.method = HTypes.methodPost
381+ , HTTP.requestBody = HTTP.RequestBodyLBS (BB.toLazyByteString (bulkBuffer_payload bulkBuffer))
382+ }
383+
384+ retryPolicy = logzIOScribeConfiguration_retry config
385+ apiTokenBS = TE.encodeUtf8 (apiToken (logzIOScribeConfiguration_token config))
386+
387+ handleHttpException :: (Applicative m) => HTTP.HttpException -> m Bool
388+ handleHttpException _ = pure True
389+
390+ uriStr = LBS8.unpack (BB.toLazyByteString (URIBS.serializeURIRef uri))
391+
392+ authority = URIBS.Authority
393+ { URIBS.authorityUserInfo = Nothing
394+ , URIBS.authorityHost = logzIOScribeConfiguration_host config
395+ , URIBS.authorityPort = Just (logzIOScribeConfiguration_port config)
396+ }
397+
398+ uri = URIBS.URI
399+ { URIBS.uriScheme = case logzIOScribeConfiguration_scheme config of
400+ HTTPS -> URIBS.Scheme "https"
401+ HTTP -> URIBS.Scheme "http"
402+ , URIBS.uriAuthority = Just authority
403+ , URIBS.uriPath = "/"
404+ , URIBS.uriQuery = URIBS.Query
405+ [ ("token", apiTokenBS)
406+ ]
407+ , URIBS.uriFragment = Nothing
408+ }
409+
410+
411+-------------------------------------------------------------------------------
412+newtype Bytes = Bytes
413+ { bytes :: Int64
414+ } deriving (Show, Eq, Num, Ord, Bounded)
415+
416+
417+-- | How big of a body can we send? The limit is defined
418+-- <https://app.logz.io/#/dashboard/data-sources/Bulk-HTTPS here>.
419+maxPayloadBytes :: Bytes
420+maxPayloadBytes = 10485760
421+
422+
423+-- | How long can each serialized payload be (let's assume including
424+-- the trailing newline). The limit is defined
425+-- <https://app.logz.io/#/dashboard/data-sources/Bulk-HTTPS here>.
426+maxLogLineLength :: Bytes
427+maxLogLineLength = 500000
428+
429+
430+measureJSONLine :: A.ToJSON a => a -> (BB.Builder, Bytes)
431+measureJSONLine a = (BB.lazyByteString lbs, Bytes (LBS.length lbs))
432+ where
433+ lbs = A.encode a <> "\n"
434+
435+
436+-- | Fully-rendered JSON object for an item
437+fullItemObject :: K.LogItem a => K.Verbosity -> K.Item a -> A.Object
438+fullItemObject verbosity item = HM.fromList
439+ [ "app" A..= K._itemApp item
440+ , "env" A..= K._itemEnv item
441+ , "sev" A..= K._itemSeverity item
442+ , "thread" A..= K.getThreadIdText (K._itemThread item)
443+ , "host" A..= K._itemHost item
444+ , "pid" A..= pidInt
445+ , "data" A..= annotateKeys (K.payloadObject verbosity (K._itemPayload item))
446+ -- Slight deviation, logz.io uses "message" instead of "msg"
447+ , "message" A..= TB.toLazyText (K.unLogStr (K._itemMessage item))
448+ -- Another slight deviation, logz.io uses "@timestamp" instead of
449+ -- "at". Note, your logs should be sent roughly close to when they
450+ -- are created. They are assigned to indexes based on index date, so
451+ -- time searches act strange if you backfill too far from the
452+ -- past. They seem to support 3 decimal places of
453+ -- precision. Formatting like this requires time 1.8.0.2, which is
454+ -- reflected in the cabal file.
455+ , "@timestamp" A..= A.String (T.pack (Time.formatTime Time.defaultTimeLocale "%Y-%m-%dT%H:%M:%S%03QZ" (K._itemTime item)))
456+ , "ns" A..= K._itemNamespace item
457+ , "loc" A..= (LocJs <$> K._itemLoc item)
458+ ]
459+ where
460+ POSIX.CPid pidInt = K._itemProcess item
461+
462+
463+-- | A version of 'renderLine' which renders a line and stays under
464+-- the maximum line size of 500,000 bytes. If the default rendering is
465+-- too large, the log will be reduced to a timestamp and a potentially
466+-- truncated message.
467+renderLineTruncated :: K.LogItem a => K.Verbosity -> K.Item a -> (BB.Builder, Bytes)
468+renderLineTruncated = renderLineTruncated' maxLogLineLength
469+
470+
471+-- | A generalized renderLineTruncated that takes a custom line length
472+-- limit. This is exclusively for testing.
473+renderLineTruncated'
474+ :: K.LogItem a
475+ => Bytes
476+ -- ^ Custom max log line length. Be careful, too low and no amount
477+ -- of shrinking can get under the size, breaking the invariant. For
478+ -- the production limit, we are guraanteed always able to come in
479+ -- under the limit.
480+ -> K.Verbosity
481+ -> K.Item a
482+ -> (BB.Builder, Bytes)
483+renderLineTruncated' customMaxLogLineLength verbosity item =
484+ if fullSize <= customMaxLogLineLength
485+ then (fullLine, fullSize)
486+ else (fallbackLine, fallbackSize)
487+ where
488+ fullObject = fullItemObject verbosity item
489+ (fullLine, fullSize) = measureJSONLine fullObject
490+ -- only the absolutely necessary keys, with message stripped out
491+ -- to help us calculate how much of the message we can keep. these
492+ -- are lazily evaluated and as such won't be computed unless the
493+ -- item is too big
494+ blankObject :: A.Object
495+ blankObject = HM.fromList
496+ [ "message" A..= A.String "" -- we'll start with a blank message
497+ , "@timestamp" A..= K._itemTime item
498+ ]
499+ (_, blankObjectSize) = measureJSONLine blankObject
500+ messageBytesAllowed = maxLogLineLength - blankObjectSize
501+ (fallbackLine, fallbackSize) = measureJSONLine fallbackObject
502+ fallbackObject :: A.Object
503+ fallbackObject = HM.fromList
504+ [ "message" A..= A.toJSON (TL.take (bytes messageBytesAllowed) (TB.toLazyText (K.unLogStr (K._itemMessage item))))
505+ , "@timestamp" A..= A.toJSON (K._itemTime item)
506+ ]
507+
508+
509+data BulkBuffer = BulkBuffer
510+ { bulkBuffer_bytesUsed :: !Bytes
511+ , bulkBuffer_payload :: !BB.Builder
512+ , bulkBuffer_itemCount :: !Int
513+ }
514+
515+
516+instance Semigroup.Semigroup BulkBuffer where
517+ (BulkBuffer bytesUsedA bufferA itemCountA) <>
518+ (BulkBuffer bytesUsedB bufferB itemCountB) = BulkBuffer
519+ (bytesUsedA + bytesUsedB)
520+ (bufferA <> bufferB)
521+ (itemCountA + itemCountB)
522+
523+
524+instance Monoid BulkBuffer where
525+ mempty = BulkBuffer 0 mempty 0
526+ mappend = (<>)
527+
528+
529+data LogAction buf =
530+ Buffered
531+ buf
532+ -- ^ New buffer
533+ | FlushNow
534+ buf
535+ -- ^ Buffer to flush
536+ buf
537+ -- ^ New buffer
538+
539+bufferItem
540+ :: K.LogItem a
541+ => Int
542+ -- ^ Maximum items before flushing
543+ -> K.Verbosity
544+ -> K.Item a
545+ -> BulkBuffer
546+ -> LogAction BulkBuffer
547+bufferItem = bufferItem' maxPayloadBytes maxLogLineLength
548+
549+
550+-- | An internal version with a configurable max item size and max
551+-- message size for testing. Be careful with this: if you set the
552+-- values too low, some of the guarantees about reducability of
553+-- messages break down.
554+bufferItem'
555+ :: (K.LogItem a)
556+ => Bytes
557+ -- ^ Max payload size
558+ -> Bytes
559+ -- ^ Max item size
560+ -> Int
561+ -- ^ Maximum items before flushing
562+ -> K.Verbosity
563+ -> K.Item a
564+ -> BulkBuffer
565+ -> LogAction BulkBuffer
566+bufferItem' customMaxPayload customMaxItem maxItems verb item bulkBuffer =
567+ let (encodedLine, spaceNeeded) = renderLineTruncated' customMaxItem verb item
568+ newBytesUsed = bulkBuffer_bytesUsed bulkBuffer + spaceNeeded
569+ newItemCount = bulkBuffer_itemCount bulkBuffer + 1
570+ in if newItemCount >= maxItems || newBytesUsed >= customMaxPayload
571+ then FlushNow
572+ bulkBuffer
573+ BulkBuffer
574+ { bulkBuffer_bytesUsed = spaceNeeded
575+ , bulkBuffer_payload = encodedLine
576+ , bulkBuffer_itemCount = 1
577+ }
578+ else Buffered $
579+ BulkBuffer
580+ { bulkBuffer_bytesUsed = newBytesUsed
581+ , bulkBuffer_payload = bulkBuffer_payload bulkBuffer <> encodedLine
582+ , bulkBuffer_itemCount = newItemCount
583+ }
584+
585+
586+-- | When time has run out on a flush, splits the buffer. n.b. this
587+-- will always return 'FlushNow' with an empty new buffer.
588+forceFlush :: (Monoid buf) => buf -> LogAction buf
589+forceFlush buf = FlushNow buf mempty
590+
591+
592+-------------------------------------------------------------------------------
593+-- Annotation borrowed from katip-elasticsearch. There are fixed
594+-- fields in katip logs which should stay the same type forever and
595+-- thus won't need annotation. However, any code in userland may
596+-- choose to add akey to their log's metadata with a type that's
597+-- incompatible with the mapping. If the first value that's picked up
598+-- restrictive (e.g. a long), subsequent values will be rejected. By
599+-- differentiating types by their name, this is guaranteed not to
600+-- happen.
601+
602+annotateValue :: A.Value -> A.Value
603+annotateValue (A.Object o) = A.Object (annotateKeys o)
604+annotateValue (A.Array a) = A.Array (annotateValue <$> a)
605+annotateValue x = x
606+
607+
608+annotateKeys :: A.Object -> A.Object
609+annotateKeys = HM.fromList . map go . HM.toList
610+ where
611+ go (k, A.Object o) = (k, A.Object (annotateKeys o))
612+ go (k, A.Array a) = (k, A.Array (annotateValue <$> a))
613+ go (k, s@(A.String _)) = (k <> stringAnn, s)
614+ go (k, n@(A.Number sci)) = if Scientific.isFloating sci
615+ then (k <> doubleAnn, n)
616+ else (k <> longAnn, n)
617+ go (k, b@(A.Bool _)) = (k <> booleanAnn, b)
618+ go (k, A.Null) = (k <> nullAnn, A.Null)
619+
620+
621+-------------------------------------------------------------------------------
622+-- Annotation Constants
623+-------------------------------------------------------------------------------
624+
625+
626+stringAnn :: T.Text
627+stringAnn = "::s"
628+
629+doubleAnn :: T.Text
630+doubleAnn = "::d"
631+
632+longAnn :: T.Text
633+longAnn = "::l"
634+
635+booleanAnn :: T.Text
636+booleanAnn = "::b"
637+
638+nullAnn :: T.Text
639+nullAnn = "::n"
katip-logzio/test/Katip/Tests/Scribes/LogzIO/HTTPS.hsnew file+466
@@ -0,0 +1,466 @@
1+{-# LANGUAGE OverloadedStrings #-}
2+module Katip.Tests.Scribes.LogzIO.HTTPS
3+ ( tests
4+ ) where
5+
6+
7+-------------------------------------------------------------------------------
8+import Control.Concurrent.Async
9+import Control.Concurrent.STM
10+import Control.Concurrent.STM.TSem
11+import Control.Exception.Safe
12+import Control.Monad
13+import Control.Monad.IO.Class
14+import Data.Aeson as A
15+import qualified Data.ByteString.Builder as BB
16+import qualified Data.ByteString.Lazy as LBS
17+import qualified Data.ByteString.Lazy.Char8 as LBS8
18+import qualified Data.HashMap.Strict as HM
19+import Data.Int
20+import Data.Scientific
21+import Data.Text (Text)
22+import Data.Time
23+import Data.Time.Clock.POSIX
24+import qualified Data.Vector as V
25+import Hedgehog as HH
26+import qualified Hedgehog.Gen as Gen
27+import qualified Hedgehog.Range as Range
28+import Katip
29+import Katip.Core
30+import Language.Haskell.TH (Loc (..))
31+import Network.HostName
32+import qualified Network.HTTP.Types.Status as HTypes
33+import qualified Network.Wai.Handler.Warp as Warp
34+import System.Posix
35+import System.Timeout
36+import Test.Tasty
37+import Test.Tasty.Hedgehog
38+import Test.Tasty.HUnit
39+import URI.ByteString
40+import qualified Web.Scotty as Scotty
41+-------------------------------------------------------------------------------
42+import Katip.Scribes.LogzIO.HTTPS
43+-------------------------------------------------------------------------------
44+
45+
46+tests :: TestTree
47+tests = testGroup "Katip.Scribes.LogzIO.HTTP"
48+ [ bufferItemTests
49+ , renderLineTruncatedTests
50+ , scribeTests
51+ ]
52+
53+
54+-------------------------------------------------------------------------------
55+-- We bind to a system port so we use a lock on these tests to avoid interleaved, concurrent requests
56+scribeTests :: TestTree
57+scribeTests = withLock $ \mkSem -> testGroup "scribe"
58+ [ testCase "writes well-formed log items as lines to the given API" $ do
59+ sem <- mkSem
60+ withServer sem $ \receivedMessages -> do
61+ withScribe (\cfg -> cfg { logzIOScribeConfiguration_bufferTimeout = 1}) $ \scribe -> do
62+ (liPush scribe) (genericLog "a message")
63+
64+ requestLog <- timeoutFailure (20 * second) (atomically (readTQueue receivedMessages))
65+ requestLog_token requestLog @?= testAPIToken
66+ let logs = requestLog_logs requestLog
67+ length logs @?= 1
68+ let firstLog = head logs
69+ receivedLog_message firstLog @?= "a message"
70+ , testCase "respects the items limit" $ do
71+ sem <- mkSem
72+ withServer sem $ \receivedMessages -> do
73+ withScribe (\cfg -> cfg { logzIOScribeConfiguration_bufferItems = 1, logzIOScribeConfiguration_bufferTimeout = 1}) $ \scribe -> do
74+ (liPush scribe) (genericLog "message 1")
75+ (liPush scribe) (genericLog "message 2")
76+
77+ log1 <- timeoutFailure (20 * second) (atomically (readTQueue receivedMessages))
78+ requestLog_token log1 @?= testAPIToken
79+ let logs1 = requestLog_logs log1
80+ length logs1 @?= 1
81+ let firstLog1 = head logs1
82+ receivedLog_message firstLog1 @?= "message 1"
83+
84+ log2 <- timeoutFailure (20 * second) (atomically (readTQueue receivedMessages))
85+ let logs2 = requestLog_logs log2
86+ length logs2 @?= 1
87+ let firstLog2 = head logs2
88+ receivedLog_message firstLog2 @?= "message 2"
89+ , testCase "respects the time limit" $ do
90+ sem <- mkSem
91+ withServer sem $ \receivedMessages -> do
92+ withScribe (\cfg -> cfg { logzIOScribeConfiguration_bufferItems = 2, logzIOScribeConfiguration_bufferTimeout = 1}) $ \scribe -> do
93+ (liPush scribe) (genericLog "a message")
94+ log1 <- timeoutFailure (20 * second) (atomically (readTQueue receivedMessages))
95+ requestLog_token log1 @?= testAPIToken
96+ let logs1 = requestLog_logs log1
97+ length logs1 @?= 1
98+ let firstLog1 = head logs1
99+ receivedLog_message firstLog1 @?= "a message"
100+ , testCase "reports errors" $ do
101+ sem <- mkSem
102+ let badToken = APIToken "badtoken"
103+ errorSignal <- newEmptyTMVarIO
104+ withServer sem $ \_receivedMessages -> do
105+ withScribe (\cfg -> cfg { logzIOScribeConfiguration_bufferItems = 1, logzIOScribeConfiguration_bufferTimeout = 1, logzIOScribeConfiguration_token = badToken, logzIOScribeConfiguration_onError = void . atomically . tryPutTMVar errorSignal}) $ \scribe -> do
106+ (liPush scribe) (genericLog "a message")
107+ loggingError <- timeoutFailure (20 * second) (atomically (takeTMVar errorSignal))
108+ case loggingError of
109+ BadToken -> pure ()
110+ _ -> assertFailure ("Expected BadToken error but got " <> show loggingError)
111+ , testCase "flushes all queued logs on close" $ do
112+ sem <- mkSem
113+ withServer sem $ \receivedMessages -> do
114+ withScribe (\cfg -> cfg { logzIOScribeConfiguration_bufferItems = 100, logzIOScribeConfiguration_bufferTimeout = 100}) $ \scribe -> do
115+ (liPush scribe) (genericLog "message 1")
116+ (liPush scribe) (genericLog "message 2")
117+ messages <- atomically (flushTQueue receivedMessages)
118+ let allLogs = receivedLog_message <$> mconcat (requestLog_logs <$> messages)
119+ allLogs @?= ["message 1", "message 2"]
120+ ]
121+ where
122+ second = 1000000
123+ timeoutFailure waitTime f = do
124+ res <- timeout waitTime f
125+ case res of
126+ Nothing -> assertFailure ("Operation timed out after " <> show waitTime <> "us")
127+ Just x -> pure x
128+ withLock = withResource (atomically (newTSem 1)) (const (return ()))
129+ withServer sem f = bracket_ (atomically (waitTSem sem)) (atomically (signalTSem sem)) $ do
130+ receivedMessages <- atomically newTQueue
131+ withAsync (startServer receivedMessages) $ \_async -> f receivedMessages
132+ testAPIToken = "bigSECRET"
133+ genericLog message = Item
134+ { _itemApp = "katip-logzio"
135+ , _itemEnv = "test"
136+ , _itemSeverity = InfoS
137+ , _itemThread = ThreadIdText "42"
138+ , _itemHost = "localhost"
139+ , _itemProcess = 111
140+ , _itemPayload = ()
141+ , _itemMessage = message
142+ , _itemTime = mkUTCTime 2019 1 2 3 4 5
143+ , _itemNamespace = "test"
144+ , _itemLoc = Nothing
145+ }
146+ startServer receivedMessages = Scotty.scottyOpts serverOpts $ do
147+ Scotty.post "/" $ do
148+ token <- APIToken <$> Scotty.param "token"
149+ if token == testAPIToken
150+ then do
151+ bodyLines <- LBS8.lines <$> Scotty.body
152+ receivedLogs <- mapM (either error pure . A.eitherDecode) bodyLines
153+ let requestLog = RequestLog
154+ { requestLog_token = token
155+ , requestLog_logs = receivedLogs
156+ }
157+ liftIO (atomically (writeTQueue receivedMessages requestLog))
158+ else Scotty.status HTypes.unauthorized401
159+ serverOpts = Scotty.Options
160+ { Scotty.verbose = 0
161+ , Scotty.settings = Warp.setPort 1337 Warp.defaultSettings
162+ }
163+ withScribe modConfig =
164+ bracket (mkLogzIOScribe (modConfig scribeConfig) (const (pure True)) V3) scribeFinalizer
165+ scribeConfig = LogzIOScribeConfiguration
166+ { logzIOScribeConfiguration_bufferItems = 5
167+ , logzIOScribeConfiguration_bufferTimeout = 10
168+ , logzIOScribeConfiguration_scheme = HTTP
169+ , logzIOScribeConfiguration_host = Host "127.0.0.1"
170+ , logzIOScribeConfiguration_port = Port 1337
171+ , logzIOScribeConfiguration_token = testAPIToken
172+ , logzIOScribeConfiguration_retry = defaultRetryPolicy
173+ , logzIOScribeConfiguration_onError = const (pure ())
174+ }
175+
176+
177+
178+data RequestLog = RequestLog
179+ { requestLog_token :: APIToken
180+ , requestLog_logs :: [ReceivedLog]
181+ }
182+
183+
184+-- | Simplified log that only parses the fields we want to assert on
185+data ReceivedLog = ReceivedLog
186+ { receivedLog_message :: Text
187+ }
188+
189+
190+instance FromJSON ReceivedLog where
191+ parseJSON = withObject "ReceivedLog" $ \o -> do
192+ message <- o .: "message"
193+ pure $ ReceivedLog
194+ { receivedLog_message = message
195+ }
196+
197+
198+-------------------------------------------------------------------------------
199+bufferItemTests :: TestTree
200+bufferItemTests = testGroup "bufferItem"
201+ [ testProperty "0 or negative maximum items always flushes immediately" $ property $ do
202+ item <- forAllWith showSimplePayloadItem genSimplePayloadItem
203+ verbosity <- forAll genVerbosity
204+ maxItems <- forAll (Gen.int (Range.linear minBound 0))
205+ bulkBuffer <- forAllWith inspectBulkBuffer genBulkBuffer
206+ let logAction = bufferItem' reducedMaxPayloadBytes reducedMaxLogLineLength maxItems verbosity item bulkBuffer
207+ assertFlushNow logAction
208+ , testProperty "never buffers too many items" $ property $ do
209+ item <- forAllWith showSimplePayloadItem genSimplePayloadItem
210+ verbosity <- forAll genVerbosity
211+ maxItems <- forAll (Gen.int (Range.linear 0 100))
212+ bulkBuffer <- forAllWith inspectBulkBuffer genBulkBuffer
213+ let logAction = bufferItem' reducedMaxPayloadBytes reducedMaxLogLineLength maxItems verbosity item bulkBuffer
214+ case logAction of
215+ Buffered newBuffer -> HH.assert (bulkBuffer_itemCount newBuffer <= maxItems)
216+ _ -> pure ()
217+ , testProperty "never produces too many bytes" $ property $ do
218+ item <- forAllWith showSimplePayloadItem genSimplePayloadItem
219+ verbosity <- forAll genVerbosity
220+ maxItems <- forAll (Gen.int (Range.linear 0 100))
221+ bulkBuffer <- forAllWith inspectBulkBuffer genBulkBuffer
222+ let logAction = bufferItem' reducedMaxPayloadBytes reducedMaxLogLineLength maxItems verbosity item bulkBuffer
223+ case logAction of
224+ Buffered newBuffer -> HH.assert (bulkBuffer_bytesUsed newBuffer <= reducedMaxPayloadBytes)
225+ _ -> pure ()
226+ , testProperty "never flushes more than 1 item" $ property $ do
227+ item <- forAllWith showSimplePayloadItem genSimplePayloadItem
228+ verbosity <- forAll genVerbosity
229+ maxItems <- forAll (Gen.int (Range.linear minBound 0))
230+ bulkBuffer <- forAllWith inspectBulkBuffer genBulkBuffer
231+ let logAction = bufferItem' reducedMaxPayloadBytes reducedMaxLogLineLength maxItems verbosity item bulkBuffer
232+ case logAction of
233+ FlushNow _ newBuffer -> bulkBuffer_itemCount newBuffer === 1
234+ _ -> pure ()
235+ , testProperty "flushes when the buffer is exactly at the limit" $ property $ do
236+ maxItems <- forAll (Gen.int (Range.linear 1 20))
237+ item <- forAllWith showSimplePayloadItem genSimplePayloadItem
238+ bulkBuffer <- forAllWith inspectBulkBuffer ((\b -> b { bulkBuffer_itemCount = maxItems - 1 }) <$> genBulkBuffer)
239+ verbosity <- forAll genVerbosity
240+ let logAction = bufferItem' maxBound maxBound maxItems verbosity item bulkBuffer
241+ case logAction of
242+ FlushNow _ _ -> pure ()
243+ Buffered _ -> fail "Expected a FlushNow but got a Buffered"
244+ ]
245+ where
246+ assertFlushNow (FlushNow _ _) = pure ()
247+ assertFlushNow (Buffered _) = fail ("Expceted FlushNow but got a Buffered")
248+ inspectBulkBuffer (BulkBuffer used _ itemCount) =
249+ "BulkBuffer (" <> show used <> ") ... " <> show itemCount
250+
251+
252+-------------------------------------------------------------------------------
253+-- | Reduced max log line length for more space and time efficient tests
254+reducedMaxLogLineLength :: Bytes
255+reducedMaxLogLineLength = Bytes (bytes maxLogLineLength `div` limitScalingFactor)
256+
257+
258+reducedMaxPayloadBytes :: Bytes
259+reducedMaxPayloadBytes = Bytes (bytes maxPayloadBytes `div` limitScalingFactor)
260+
261+
262+limitScalingFactor :: Int64
263+limitScalingFactor = 10
264+
265+-------------------------------------------------------------------------------
266+showSimplePayloadItem :: Item SimpleLogPayload -> String
267+showSimplePayloadItem item =
268+ show (item { _itemPayload = [(k, toJSON v) | (k, AnyLogPayload v) <- unSimpleLogPayload (_itemPayload item)]})
269+
270+
271+-------------------------------------------------------------------------------
272+renderLineTruncatedTests :: TestTree
273+renderLineTruncatedTests = testGroup "renderLineTruncated"
274+ [ testProperty "always produces an accurate byte count" $ property $ do
275+ item <- forAllWith showSimplePayloadItem genSimplePayloadItem
276+ verbosity <- forAll genVerbosity
277+ let (builder, Bytes byteCount) = renderLineTruncated' reducedMaxLogLineLength verbosity item
278+ builderLength builder === byteCount
279+ , testProperty "always produces a payload smaller than the line limit" $ property $ do
280+ let tinyMaxLineLength = 1024 -- set a really low one for speed
281+ let lowerBound = fromIntegral (bytes tinyMaxLineLength - 100)
282+ let upperBound = fromIntegral (bytes tinyMaxLineLength + 100)
283+ bigMessage <- forAll (genLogStr' (Range.linear lowerBound upperBound))
284+ item <- (\item -> item { _itemMessage = bigMessage}) <$> forAllWith showSimplePayloadItem genSimplePayloadItem
285+ verbosity <- forAll genVerbosity
286+ let (_, byteCount) = renderLineTruncated' tinyMaxLineLength verbosity item
287+ HH.assert (byteCount <= reducedMaxLogLineLength)
288+ ]
289+
290+
291+-------------------------------------------------------------------------------
292+builderLength :: BB.Builder -> Int64
293+builderLength = LBS.length . BB.toLazyByteString
294+
295+-------------------------------------------------------------------------------
296+genVerbosity :: Gen Verbosity
297+genVerbosity = Gen.enumBounded
298+
299+
300+-------------------------------------------------------------------------------
301+genBulkBuffer :: Gen BulkBuffer
302+genBulkBuffer = do
303+ bytesUsed <- genBytes
304+ -- generating a large bytestring string with hedghog seems quite slow, instead we'll generate a smaller string and replicate and merge it
305+ payloadMultiplier <- Gen.int (Range.linear 1 100)
306+ payloadSegment <- genBuilder (Range.linear 0 (round (toRational (bytes reducedMaxPayloadBytes) * 0.05)))
307+ let payload = mconcat (replicate payloadMultiplier payloadSegment)
308+ itemCount <- Gen.int (Range.linear 0 maxBound)
309+ pure $ BulkBuffer
310+ { bulkBuffer_bytesUsed = bytesUsed
311+ , bulkBuffer_payload = payload
312+ , bulkBuffer_itemCount = itemCount
313+ }
314+
315+
316+-------------------------------------------------------------------------------
317+genBytes :: Gen Bytes
318+genBytes = Bytes <$> Gen.int64 (Range.linear 0 maxBound)
319+
320+
321+-------------------------------------------------------------------------------
322+genBuilder :: Range Int -> Gen BB.Builder
323+genBuilder size = BB.byteString <$> Gen.bytes size
324+
325+
326+-------------------------------------------------------------------------------
327+genSimplePayloadItem :: Gen (Item SimpleLogPayload)
328+genSimplePayloadItem = do
329+ app <- genNamespace
330+ env <- genEnvironment
331+ severity <- genSeverity
332+ thread <- genThreadIdText
333+ host <- genHostName
334+ process <- genProcessID
335+ payload <- genSimpleLogPayload
336+ message <- genLogStr
337+ time <- genUTCTime
338+ namespace <- genNamespace
339+ loc <- Gen.maybe genLoc
340+ pure $ Item
341+ { _itemApp = app
342+ , _itemEnv = env
343+ , _itemSeverity = severity
344+ , _itemThread = thread
345+ , _itemHost = host
346+ , _itemProcess = process
347+ , _itemPayload = payload
348+ , _itemMessage = message
349+ , _itemTime = time
350+ , _itemNamespace = namespace
351+ , _itemLoc = loc
352+ }
353+
354+
355+genNamespace :: Gen Namespace
356+genNamespace = Namespace <$> Gen.list (Range.linear 0 20) genSmallText
357+
358+
359+genEnvironment :: Gen Environment
360+genEnvironment = Environment <$> genSmallText
361+
362+
363+genSeverity :: Gen Severity
364+genSeverity = Gen.enumBounded
365+
366+
367+genThreadIdText :: Gen ThreadIdText
368+genThreadIdText = ThreadIdText <$> genSmallText
369+
370+
371+genHostName :: Gen HostName
372+genHostName = genSmallString
373+
374+
375+genProcessID :: Gen ProcessID
376+genProcessID = Gen.enumBounded
377+
378+
379+genLogStr :: Gen LogStr
380+genLogStr = genLogStr' (Range.linear 0 1024)
381+
382+
383+genLogStr' :: Range Int -> Gen LogStr
384+genLogStr' sizeRange = ls <$> Gen.text sizeRange Gen.unicode
385+
386+
387+genUTCTime :: Gen UTCTime
388+genUTCTime = posixSecondsToUTCTime . fromIntegral
389+ <$> Gen.int (Range.linear 0 maxBound)
390+
391+
392+genLoc :: Gen Loc
393+genLoc = Loc
394+ <$> genSmallString
395+ <*> genSmallString
396+ <*> genSmallString
397+ <*> genCharPos
398+ <*> genCharPos
399+ where
400+ genCharPos = (,)
401+ <$> Gen.int (Range.linear 0 maxBound)
402+ <*> Gen.int (Range.linear 0 maxBound)
403+
404+
405+genSimpleLogPayload :: Gen SimpleLogPayload
406+genSimpleLogPayload = SimpleLogPayload
407+ <$> Gen.list (Range.linear 0 20) genPair
408+ where
409+ genPair = (,)
410+ <$> genSmallText
411+ <*> genAnyLogPayload
412+
413+
414+genAnyLogPayload :: Gen AnyLogPayload
415+genAnyLogPayload = AnyLogPayload
416+ <$> genValue
417+
418+
419+genValue :: Gen Value
420+genValue = Gen.recursive Gen.choice nonRecursiveGens recursiveGens
421+ where
422+ nonRecursiveGens =
423+ [ pure A.Null
424+ , A.String <$> genSmallText
425+ , A.Number <$> genScientific
426+ , A.Bool <$> Gen.bool
427+ ]
428+ recursiveGens =
429+ [ A.Object <$> genObject
430+ , A.Array <$> genArray
431+ ]
432+ genObject = HM.fromList <$> Gen.list (Range.linear 0 20) genPair
433+ where
434+ genPair = (,) <$> genSmallText <*> genValue
435+ genArray = V.fromList <$>
436+ Gen.list (Range.linear 0 20) genValue
437+
438+
439+
440+genScientific :: Gen Scientific
441+genScientific = scientific
442+ <$> Gen.integral (Range.linear (-999999) 999999)
443+ <*> Gen.int (Range.linear (-999) (999))
444+
445+
446+genSmallText :: Gen Text
447+genSmallText = Gen.text (Range.linear 0 128) Gen.unicode
448+
449+
450+genSmallString :: Gen String
451+genSmallString = Gen.string (Range.linear 0 128) Gen.unicode
452+
453+
454+-------------------------------------------------------------------------------
455+mkUTCTime :: Integer -> Int -> Int -> DiffTime -> DiffTime -> DiffTime -> UTCTime
456+mkUTCTime y m d hr mn s = UTCTime day dt
457+ where
458+ day = mkDay y m d
459+ dt = s + 60 * mn + 60 * 60 * hr
460+
461+
462+-------------------------------------------------------------------------------
463+mkDay :: Integer -> Int -> Int -> Day
464+mkDay y m d = day
465+ where
466+ Just day = fromGregorianValid y m d
katip-logzio/test/Main.hsnew file+22
@@ -0,0 +1,22 @@
1+module Main
2+ ( main
3+ ) where
4+
5+
6+-------------------------------------------------------------------------------
7+import Test.Tasty
8+-------------------------------------------------------------------------------
9+import qualified Katip.Tests.Scribes.LogzIO.HTTPS
10+-------------------------------------------------------------------------------
11+
12+
13+
14+main :: IO ()
15+main = defaultMain tests
16+
17+
18+-------------------------------------------------------------------------------
19+tests :: TestTree
20+tests = testGroup "katip-logzio"
21+ [ Katip.Tests.Scribes.LogzIO.HTTPS.tests
22+ ]
katip/README.md+4−1
@@ -16,7 +16,7 @@ Kâtip (pronounced kah-tip) is the Turkish word for scribe.
16 add context that will be automatically merged in with existing log
16 add context that will be automatically merged in with existing log
17 context.
17 context.
18
18
19−* *Easy to Integration:* Katip was designed to be easily integrated
19+* *Easy Integration:* Katip was designed to be easily integrated
20 into existing monads. By using typeclasses for logging facilities,
20 into existing monads. By using typeclasses for logging facilities,
21 individual subsystems and even libraries can easily add their own
21 individual subsystems and even libraries can easily add their own
22 namespacing and context without having any knowledge of their
22 namespacing and context without having any knowledge of their
@@ -84,3 +84,6 @@ stack.
84* [Leonid Onokhov](https://github.com/sopvop)
84* [Leonid Onokhov](https://github.com/sopvop)
85* [Alexander Vershilov](https://github.com/qnikst)
85* [Alexander Vershilov](https://github.com/qnikst)
86* [Chris Martin](https://github.com/chris-martin)
86* [Chris Martin](https://github.com/chris-martin)
87+* [Domen Kožar](https://github.com/domenkozar)
88+* [Tristan Bull](https://github.com/tmbull)
89+* [Aleksey Khudyakov](https://github.com/Shimuuar)
katip/bench/Main.hs+5−2
@@ -92,7 +92,7 @@ setup dest = do
92 return (tmp </> "katip-bench.log")
92 return (tmp </> "katip-bench.log")
93 DevNull -> return ("/dev/null")
93 DevNull -> return ("/dev/null")
94 h <- openFile outFile WriteMode
94 h <- openFile outFile WriteMode
95− s <- mkHandleScribe ColorIfTerminal h DebugS V0
95+ s <- mkHandleScribe ColorIfTerminal h (permitItem DebugS) V0
96 let cleanupHandle = hClose h `finally` (when (dest == TempFile) (removeLink outFile))
96 let cleanupHandle = hClose h `finally` (when (dest == TempFile) (removeLink outFile))
97 return s { scribeFinalizer = scribeFinalizer s `finally` cleanupHandle}
97 return s { scribeFinalizer = scribeFinalizer s `finally` cleanupHandle}
98
98
@@ -101,4 +101,7 @@ setup dest = do
101deriving instance NFData ThreadIdText
101deriving instance NFData ThreadIdText
102
102
103instance NFData Scribe where
103instance NFData Scribe where
104− rnf (Scribe a b) = (a :: Item ExPayload -> IO ()) `seq` b `seq` ()
104+ rnf (Scribe a b p) = (a :: Item ExPayload -> IO ())
105+ `seq` b
106+ `seq` (p :: Item ExPayload -> IO Bool)
107+ `seq` ()
katip/changelog.md+86
@@ -1,3 +1,89 @@
1+0.8.5.0
2+=======
3+* Add MonadResource instances [#121](https://github.com/Soostone/katip/pull/121)
4+
5+0.8.4.0
6+=======
7+* Add support for unliftio-core 0.2 while maintaining backwards-compatible support for 0.1
8+
9+0.8.3.0
10+=======
11+
12+* Add `logKatipItem` function and reimplement `logItem` to use it. [vlatkoB](https://github.com/vlatkoB)
13+
14+0.8.2.0
15+=======
16+* Add `MonadFail` instances for `base` >= 4.9.0.0 [Koray Al](https://github.com/korayal)
17+
18+0.8.1.0
19+=======
20+* Export `logLoc`. Credit to [Brian McKenna](https://github.com/puffnfresh)
21+
22+0.8.0.0
23+=======
24+* **Breaking change**: switch from severity based log level filtering to
25+ user-definable `PermitFunc`. To upgrade, any time you initialize one
26+ of the standard scribes, rather than providing a severity like
27+ `InfoS`, instead pass in `(permitItem InfoS)`. Credit to
28+ [vlatkoB](https://github.com/vlatkoB).
29+* Update some misformatted and outdated docs.
30+* Set lower bound of base to 4.9.0.0, the version shipped with GHC
31+ 8.0.2. Support for GHC versions older than that is removed.
32+* Update default formatter of Handle scribe to format pid as `[PID
33+ 1234]` instead of `[1234]` and `[ThreadId 1234]` instead of
34+ `[1234]`.
35+* Add missing `Bounded` instance for `Verbosity`. This allows you to
36+ easily enumerate all verbosities via `[minBound..maxBound]`.
37+
38+0.7.0.0
39+=======
40+* Exclusively use HasCallstack for location reporting in errors. Credit to [Domen Kožar](https://github.com/domenkozar)
41+* Drop support for GHC 7.x
42+
43+0.6.3.0
44+=======
45+* Add `ToJSON`/`FromJSON` instances for Verbosity. Credit to [Aleksey Khudyakov](https://github.com/Shimuuar).
46+
47+0.6.2.1
48+=======
49+* Add `mkHandleScribeWithFormatter`. This allows control over the format of log items going to a handle. Credit to [Tristan Bull](https://github.com/tmbull) for the implementation.
50+* Documentation fixes. Credit to [Domen Kožar](https://github.com/domenkozar) for the implementation.
51+* Add `jsonFormat` handle formatter.
52+* Deprecate `formatItem` which is now replaced by `bracketFormat`.
53+
54+0.6.1.0
55+=======
56+* Loosen deps
57+* Support latest STM
58+
59+0.6.0.0
60+=======
61+* Drop ListT instance due to deprecations.
62+* Loosen dependencies.
63+
64+0.5.5.1
65+=======
66+* Decrease default timer resolution from 1s to 1ms.
67+
68+0.5.5.0
69+=======
70+* Export `runNoLoggingT` and `NoLoggingT` constructor.
71+* Delegate MonadReader instance for `NoLoggingT`.
72+
73+0.5.4.0
74+=======
75+* Loosen bounds on resourcet and template-haskell.
76+* Add convenience function askLoggerIO.
77+
78+0.5.3.0
79+=======
80+* Add MonadUnliftIO instances.
81+* Add NoLoggingT
82+
83+0.5.2.0
84+=======
85+* Allow newer versions of either by conditionally adding instances for the removed EitherT interface.
86+
10.5.1.0
870.5.1.0
2=======
88=======
3* Add mkFileScribe, a specialization of mkHandleScribe for files that manages the handle automatically.
89* Add mkFileScribe, a specialization of mkHandleScribe for files that manages the handle automatically.
katip/examples/example.hs+1−1
@@ -36,7 +36,7 @@ main = do
36 -- hot-swap scribes at runtime if you need to. 'closeScribes' is
36 -- hot-swap scribes at runtime if you need to. 'closeScribes' is
37 -- blocking and flushes all messages out of a scribe and cleans up
37 -- blocking and flushes all messages out of a scribe and cleans up
38 -- resources that were allocated at creation.
38 -- resources that were allocated at creation.
39− handleScribe <- mkHandleScribe ColorIfTerminal stdout InfoS V2
39+ handleScribe <- mkHandleScribe ColorIfTerminal stdout (permitItem InfoS) V2
40 let mkLogEnv = registerScribe "stdout" handleScribe defaultScribeSettings =<< initLogEnv "MyApp" "production"
40 let mkLogEnv = registerScribe "stdout" handleScribe defaultScribeSettings =<< initLogEnv "MyApp" "production"
41 bracket mkLogEnv closeScribes $ \le -> do
41 bracket mkLogEnv closeScribes $ \le -> do
42 let s = MyState M.mempty mempty le
42 let s = MyState M.mempty mempty le
katip/examples/example_lens.hs+1−1
@@ -51,7 +51,7 @@ main = do
51 -- hot-swap scribes at runtime if you need to. 'closeScribes' is
51 -- hot-swap scribes at runtime if you need to. 'closeScribes' is
52 -- blocking and flushes all messages out of a scribe and cleans up
52 -- blocking and flushes all messages out of a scribe and cleans up
53 -- resources that were allocated at creation.
53 -- resources that were allocated at creation.
54− handleScribe <- mkHandleScribe ColorIfTerminal stdout InfoS V2
54+ handleScribe <- mkHandleScribe ColorIfTerminal stdout (permitItem InfoS) V2
55 let mkLogEnv = registerScribe "stdout" handleScribe defaultScribeSettings =<< initLogEnv "MyApp" "production"
55 let mkLogEnv = registerScribe "stdout" handleScribe defaultScribeSettings =<< initLogEnv "MyApp" "production"
56 bracket mkLogEnv closeScribes $ \le -> do
56 bracket mkLogEnv closeScribes $ \le -> do
57 let s = MyState M.mempty mempty le
57 let s = MyState M.mempty mempty le
katip/katip.cabal+25−24
@@ -1,5 +1,5 @@
1name: katip
1name: katip
2−version: 0.5.1.0
2+version: 0.8.5.0
3synopsis: A structured logging framework.
3synopsis: A structured logging framework.
4description:
4description:
5 Katip is a structured logging framework. See README.md for more details.
5 Katip is a structured logging framework. See README.md for more details.
@@ -25,7 +25,7 @@ extra-source-files:
25 test/Katip/Tests/Scribes/Handle.hs
25 test/Katip/Tests/Scribes/Handle.hs
26 test/Katip/Tests/Scribes/Handle-text.golden
26 test/Katip/Tests/Scribes/Handle-text.golden
27 test/Katip/Tests/Format/Time.hs
27 test/Katip/Tests/Format/Time.hs
28−tested-with: GHC == 7.8.4, GHC== 7.10.3
28+tested-with: GHC == 8.0.1, GHC == 8.2.2, GHC == 8.4.3
29
29
30source-repository head
30source-repository head
31 type: git
31 type: git
@@ -53,31 +53,32 @@ library
53 TemplateHaskell
53 TemplateHaskell
54 OverloadedStrings
54 OverloadedStrings
55
55
56− build-depends: base >=4.5 && <5
56+ build-depends: base >=4.9.0.0 && <5
57− , aeson >=0.6 && <1.3
57+ , aeson >=0.6
58 , async < 3.0.0.0
58 , async < 3.0.0.0
59− , auto-update >= 0.1 && < 0.2
59+ , auto-update >= 0.1
60− , bytestring >= 0.9 && < 0.11
60+ , bytestring >= 0.9
61− , containers >=0.4 && <0.6
61+ , containers >=0.4
62− , either >= 4 && < 4.5
62+ , either >= 4
63 , safe-exceptions >= 0.1.0.0
63 , safe-exceptions >= 0.1.0.0
64− , hostname >=1.0 && <1.1
64+ , hostname >=1.0
65− , old-locale >= 1.0 && < 1.1
65+ , old-locale >= 1.0
66− , string-conv >= 0.1 && < 0.2
66+ , string-conv >= 0.1
67− , template-haskell >= 2.8 && < 2.13
67+ , template-haskell >= 2.8
68− , text >= 0.11 && <1.3
68+ , text >= 0.11
69− , time >= 1 && < 1.9
69+ , time >= 1
70− , transformers >= 0.3 && < 0.6
70+ , transformers >= 0.3
71 , transformers-compat
71 , transformers-compat
72− , unordered-containers >= 0.2 && < 0.3
72+ , unordered-containers >= 0.2
73− , monad-control >= 1.0 && < 1.1
73+ , monad-control >= 1.0
74− , mtl >= 2.0 && < 2.3
74+ , mtl >= 2.0
75− , transformers-base >= 0.3 && < 0.6
75+ , transformers-base >= 0.3
76− , resourcet >= 1.1 && < 1.2
76+ , resourcet >= 1.1
77 , scientific >= 0.3.3.0
77 , scientific >= 0.3.3.0
78− , microlens >= 0.2.0.0 && < 0.5
78+ , microlens >= 0.2.0.0
79− , microlens-th >= 0.1.0.0 && < 0.5
79+ , microlens-th >= 0.1.0.0
80 , semigroups
80 , semigroups
81+ , unliftio-core >= 0.1
81 , stm >= 2.4
82 , stm >= 2.4
82
83
83 hs-source-dirs: src
84 hs-source-dirs: src
@@ -86,7 +87,7 @@ library
86 if flag(lib-Werror)
87 if flag(lib-Werror)
87 ghc-options: -Werror
88 ghc-options: -Werror
88 if os(windows)
89 if os(windows)
89− build-depends: Win32 >=2.3 && <2.6
90+ build-depends: Win32 >=2.3 && <2.9
90 exposed-modules: Katip.Compat
91 exposed-modules: Katip.Compat
91 else
92 else
92 build-depends: unix >= 2.5 && <2.8
93 build-depends: unix >= 2.5 && <2.8
@@ -113,7 +114,7 @@ test-suite test
113 , template-haskell
114 , template-haskell
114 , text
115 , text
115 , time
116 , time
116− , time-locale-compat >= 0.1.0.1 && < 0.2
117+ , time-locale-compat >= 0.1.0.1
117 , directory
118 , directory
118 , regex-tdfa
119 , regex-tdfa
119 , unordered-containers
120 , unordered-containers
katip/src/Katip.hs+17−7
@@ -11,8 +11,8 @@
11--
11--
12-- main :: IO ()
12-- main :: IO ()
13-- main = do
13-- main = do
14−-- handleScribe <- mkHandleScribe ColorIfTerminal stdout InfoS V2
14+-- handleScribe <- mkHandleScribe ColorIfTerminal stdout (permitItem InfoS) V2
15−-- let makeLogEnv = registerScribe "stdout" handleScribe defaultScribeSettings =<< initLogEnv "MyApp" "production"
15+-- let makeLogEnv = registerScribe "stdout" handleScribe defaultScribeSettings =<< initLogEnv \"MyApp\" \"production\"
16-- -- closeScribes will stop accepting new logs, flush existing ones and clean up resources
16-- -- closeScribes will stop accepting new logs, flush existing ones and clean up resources
17-- bracket makeLogEnv closeScribes $ \le -> do
17-- bracket makeLogEnv closeScribes $ \le -> do
18-- let initialContext = () -- this context will be attached to every log in your app and merged w/ subsequent contexts
18-- let initialContext = () -- this context will be attached to every log in your app and merged w/ subsequent contexts
@@ -44,7 +44,7 @@
44--
44--
45-- newtype App m a = App {
45-- newtype App m a = App {
46-- unApp :: ReaderT Config m a
46-- unApp :: ReaderT Config m a
47−-- } deriving (Functor, Applicative, Monad) -- more instances as needed
47+-- } deriving (Functor, Applicative, Monad, MonadIO, MonadReader Config) -- these are necessary
48--
48--
49--
49--
50-- -- These instances get even easier with lenses!
50-- -- These instances get even easier with lenses!
@@ -52,7 +52,7 @@
52-- getLogEnv = asks logEnv
52-- getLogEnv = asks logEnv
53-- -- with lens:
53-- -- with lens:
54-- -- getLogEnv = view logEnv
54-- -- getLogEnv = view logEnv
55−-- localLogEnv f (App m) = App (local (\s -> s { logEnv = f (logEnv s)}) m)
55+-- localLogEnv f (App m) = App (local (\\s -> s { logEnv = f (logEnv s)}) m)
56-- -- with lens:
56-- -- with lens:
57-- -- localLogEnv f (App m) = App (local (over logEnv f) m)
57-- -- localLogEnv f (App m) = App (local (over logEnv f) m)
58--
58--
@@ -61,13 +61,13 @@
61-- getKatipContext = asks logContext
61-- getKatipContext = asks logContext
62-- -- with lens:
62-- -- with lens:
63-- -- getKatipContext = view logContext
63-- -- getKatipContext = view logContext
64−-- localKatipContext f (App m) = App (local (\s -> s { logContext = f (logContext s)}) m)
64+-- localKatipContext f (App m) = App (local (\\s -> s { logContext = f (logContext s)}) m)
65-- -- with lens:
65-- -- with lens:
66-- -- localKatipContext f (App m) = App (local (over logContext f) m)
66-- -- localKatipContext f (App m) = App (local (over logContext f) m)
67-- getKatipNamespace = asks logNamespace
67-- getKatipNamespace = asks logNamespace
68-- -- with lens:
68-- -- with lens:
69-- -- getKatipNamespace = view logNamespace
69-- -- getKatipNamespace = view logNamespace
70−-- localKatipNamespace f (App m) = App (local (\s -> s { logNamespace = f (logNamespace s)}) m)
70+-- localKatipNamespace f (App m) = App (local (\\s -> s { logNamespace = f (logNamespace s)}) m)
71-- -- with lens:
71-- -- with lens:
72-- -- localKatipNamespace f (App m) = App (local (over logNamespace f) m)
72-- -- localKatipNamespace f (App m) = App (local (over logNamespace f) m)
73--
73--
@@ -153,19 +153,22 @@ module Katip
153 , LogStr (..)
153 , LogStr (..)
154 , logStr, ls, showLS
154 , logStr, ls, showLS
155
155
156− -- ** 'Katip' LoggingFunctions
156+ -- ** 'Katip' Logging Functions
157 -- $katiplogging
157 -- $katiplogging
158 , Katip (..)
158 , Katip (..)
159 , logF
159 , logF
160 , logMsg
160 , logMsg
161 , logT
161 , logT
162+ , logLoc
162 , logItem
163 , logItem
164+ , logKatipItem
163 , logException
165 , logException
164 -- ** 'KatipContext': Logging With Context
166 -- ** 'KatipContext': Logging With Context
165 -- $katipcontextlogging
167 -- $katipcontextlogging
166 , KatipContext (..)
168 , KatipContext (..)
167 , logFM
169 , logFM
168 , logTM
170 , logTM
171+ , logLocM
169 , logItemM
172 , logItemM
170 , logExceptionM
173 , logExceptionM
171 , AnyLogContext
174 , AnyLogContext
@@ -177,10 +180,17 @@ module Katip
177
180
178 -- * Included Scribes
181 -- * Included Scribes
179 , mkHandleScribe
182 , mkHandleScribe
183+ , mkHandleScribeWithFormatter
180 , mkFileScribe
184 , mkFileScribe
181 , ColorStrategy (..)
185 , ColorStrategy (..)
186+ , ItemFormatter
187+ , bracketFormat
188+ , jsonFormat
182
189
183 -- * Tools for implementing Scribes
190 -- * Tools for implementing Scribes
191+ , PermitFunc
192+ , permitAND
193+ , permitOR
184 , permitItem
194 , permitItem
185 , payloadObject
195 , payloadObject
186 , itemJson
196 , itemJson
katip/src/Katip/Core.hs+194−90
@@ -7,7 +7,6 @@
7{-# LANGUAGE FlexibleInstances #-}
7{-# LANGUAGE FlexibleInstances #-}
8{-# LANGUAGE GADTs #-}
8{-# LANGUAGE GADTs #-}
9{-# LANGUAGE GeneralizedNewtypeDeriving #-}
9{-# LANGUAGE GeneralizedNewtypeDeriving #-}
10−{-# LANGUAGE ImplicitParams #-}
11{-# LANGUAGE MultiParamTypeClasses #-}
10{-# LANGUAGE MultiParamTypeClasses #-}
12{-# LANGUAGE OverloadedStrings #-}
11{-# LANGUAGE OverloadedStrings #-}
13{-# LANGUAGE RankNTypes #-}
12{-# LANGUAGE RankNTypes #-}
@@ -23,44 +22,53 @@
23module Katip.Core where
22module Katip.Core where
24
23
25-------------------------------------------------------------------------------
24-------------------------------------------------------------------------------
26−import Control.Applicative as A
25+import Control.Applicative as A
27import Control.AutoUpdate
26import Control.AutoUpdate
28import Control.Concurrent
27import Control.Concurrent
29−import qualified Control.Concurrent.Async as Async
28+import qualified Control.Concurrent.Async as Async
30import Control.Concurrent.STM
29import Control.Concurrent.STM
31−import qualified Control.Concurrent.STM.TBQueue as BQ
30+import qualified Control.Concurrent.STM.TBQueue as BQ
32import Control.Exception.Safe
31import Control.Exception.Safe
33−import Control.Monad (unless, void)
32+import Control.Monad (unless, void, when)
34import Control.Monad.Base
33import Control.Monad.Base
34+#if MIN_VERSION_base(4, 9, 0)
35+import qualified Control.Monad.Fail as MF
36+#endif
35import Control.Monad.IO.Class
37import Control.Monad.IO.Class
38+import Control.Monad.IO.Unlift
36import Control.Monad.Trans.Class
39import Control.Monad.Trans.Class
37import Control.Monad.Trans.Control
40import Control.Monad.Trans.Control
41+#if !MIN_VERSION_either(4, 5, 0)
38import Control.Monad.Trans.Either
42import Control.Monad.Trans.Either
43+#endif
39import Control.Monad.Trans.Except
44import Control.Monad.Trans.Except
40import Control.Monad.Trans.Maybe
45import Control.Monad.Trans.Maybe
41import Control.Monad.Trans.Reader
46import Control.Monad.Trans.Reader
42−import Control.Monad.Trans.Resource (ResourceT, transResourceT)
47+import Control.Monad.Trans.Resource (MonadResource, ResourceT,
48+ transResourceT)
43−import Control.Monad.Trans.State.Lazy (StateT, mapStateT)
49+import Control.Monad.Trans.RWS.Lazy (RWST, mapRWST)
44−import qualified Control.Monad.Trans.State.Strict as Strict (StateT, mapStateT)
50+import qualified Control.Monad.Trans.RWS.Strict as Strict (RWST, mapRWST)
45−import Control.Monad.Trans.Writer.Lazy (WriterT, mapWriterT)
51+import Control.Monad.Trans.State.Lazy (StateT, mapStateT)
46−import qualified Control.Monad.Trans.Writer.Strict as Strict (WriterT, mapWriterT)
52+import qualified Control.Monad.Trans.State.Strict as Strict (StateT, mapStateT)
47−import Control.Monad.Trans.RWS.Lazy (RWST, mapRWST)
53+import Control.Monad.Trans.Writer.Lazy (WriterT, mapWriterT)
48−import qualified Control.Monad.Trans.RWS.Strict as Strict (RWST, mapRWST)
54+import qualified Control.Monad.Trans.Writer.Strict as Strict (WriterT,
55+ mapWriterT)
49−import Data.Aeson (FromJSON (..), ToJSON (..),
56+import Data.Aeson (FromJSON (..), ToJSON (..),
50− object)
57+ object)
51−import qualified Data.Aeson as A
58+import qualified Data.Aeson as A
52−import Data.Foldable as FT
59+import Data.Foldable as FT
53−import qualified Data.HashMap.Strict as HM
60+import qualified Data.HashMap.Strict as HM
54import Data.List
61import Data.List
55−import qualified Data.Map.Strict as M
62+import qualified Data.Map.Strict as M
56−import Data.Semigroup
63+import Data.Maybe (fromMaybe)
64+import Data.Semigroup as SG
57import Data.String
65import Data.String
58import Data.String.Conv
66import Data.String.Conv
59−import Data.Text (Text)
67+import Data.Text (Text)
60−import qualified Data.Text as T
68+import qualified Data.Text as T
61−import qualified Data.Text.Lazy.Builder as B
69+import qualified Data.Text.Lazy.Builder as B
62import Data.Time
70import Data.Time
63−import GHC.Generics hiding (to)
71+import GHC.Generics hiding (to)
64#if MIN_VERSION_base(4, 8, 0)
72#if MIN_VERSION_base(4, 8, 0)
65#if !MIN_VERSION_base(4, 9, 0)
73#if !MIN_VERSION_base(4, 9, 0)
66import GHC.SrcLoc
74import GHC.SrcLoc
@@ -68,7 +76,7 @@ import GHC.SrcLoc
68import GHC.Stack
76import GHC.Stack
69#endif
77#endif
70import Language.Haskell.TH
78import Language.Haskell.TH
71−import qualified Language.Haskell.TH.Syntax as TH
79+import qualified Language.Haskell.TH.Syntax as TH
72import Lens.Micro
80import Lens.Micro
73import Lens.Micro.TH
81import Lens.Micro.TH
74import Network.HostName
82import Network.HostName
@@ -95,7 +103,7 @@ readMay s = case [x | (x,t) <- reads s, ("","") <- lex t] of
95-- IsString/OverloadedStrings, so "foo" will result in Namespace
103-- IsString/OverloadedStrings, so "foo" will result in Namespace
96-- ["foo"].
104-- ["foo"].
97newtype Namespace = Namespace { unNamespace :: [Text] }
105newtype Namespace = Namespace { unNamespace :: [Text] }
98− deriving (Eq,Show,Read,Ord,Generic,ToJSON,FromJSON,Semigroup,Monoid)
106+ deriving (Eq,Show,Read,Ord,Generic,ToJSON,FromJSON,SG.Semigroup,Monoid)
99
107
100instance IsString Namespace where
108instance IsString Namespace where
101 fromString s = Namespace [fromString s]
109 fromString s = Namespace [fromString s]
@@ -135,7 +143,7 @@ data Severity
135-- - 'V3' implies the maximum amount of payload information.
143-- - 'V3' implies the maximum amount of payload information.
136-- - Anything in between is left to the discretion of the developer.
144-- - Anything in between is left to the discretion of the developer.
137data Verbosity = V0 | V1 | V2 | V3
145data Verbosity = V0 | V1 | V2 | V3
138− deriving (Eq, Ord, Show, Read, Generic, Enum)
146+ deriving (Eq, Ord, Show, Read, Generic, Enum, Bounded)
139
147
140
148
141-------------------------------------------------------------------------------
149-------------------------------------------------------------------------------
@@ -176,6 +184,21 @@ instance FromJSON Severity where
176 Just x -> return x
184 Just x -> return x
177 Nothing -> fail $ "Invalid Severity " ++ toS t
185 Nothing -> fail $ "Invalid Severity " ++ toS t
178
186
187+instance ToJSON Verbosity where
188+ toJSON s = A.String $ case s of
189+ V0 -> "V0"
190+ V1 -> "V1"
191+ V2 -> "V2"
192+ V3 -> "V3"
193+
194+instance FromJSON Verbosity where
195+ parseJSON = A.withText "Verbosity" $ \s -> case s of
196+ "V0" -> return V0
197+ "V1" -> return V1
198+ "V2" -> return V2
199+ "V3" -> return V3
200+ _ -> fail $ "Invalid Verbosity " ++ toS s
201+
179
202
180-------------------------------------------------------------------------------
203-------------------------------------------------------------------------------
181-- | Log message with Builder underneath; use '<>' to concat in O(1).
204-- | Log message with Builder underneath; use '<>' to concat in O(1).
@@ -227,7 +250,9 @@ newtype ThreadIdText = ThreadIdText {
227
250
228
251
229mkThreadIdText :: ThreadId -> ThreadIdText
252mkThreadIdText :: ThreadId -> ThreadIdText
230−mkThreadIdText = ThreadIdText . T.pack . show
253+mkThreadIdText = ThreadIdText . stripPrefix' "ThreadId " . T.pack . show
254+ where
255+ stripPrefix' pfx t = fromMaybe t (T.stripPrefix pfx t)
231
256
232
257
233-------------------------------------------------------------------------------
258-------------------------------------------------------------------------------
@@ -515,52 +540,93 @@ itemJson verb a = toJSON $ a & itemPayload %~ payloadObject verb
515--
540--
516-- Scribes should always take a 'Severity' and 'Verbosity'.
541-- Scribes should always take a 'Severity' and 'Verbosity'.
517--
542--
518−-- Severity is used to *exclude log messages* that are < the provided
543+-- Severity is used to __exclude log messages__ that are lower than
519−-- Severity. For instance, if the user passes InfoS, DebugS items
544+-- the provided Severity. For instance, if the user passes InfoS,
520−-- should be ignored. Katip provides the 'permitItem' utility for this.
545+-- DebugS items should be ignored. Katip provides the 'permitItem'
546+-- utility for this. The user or the scribe may use 'permitAND' and
547+-- 'permitOR' to further customize this filtering, even dynamically if
548+-- they wish to.
521--
549--
522-- Verbosity is used to select keys from the log item's payload. Each
550-- Verbosity is used to select keys from the log item's payload. Each
523-- 'LogItem' instance describes what keys should be retained for each
551-- 'LogItem' instance describes what keys should be retained for each
524-- Verbosity level. Use the 'payloadObject' utility for extracting the keys
552-- Verbosity level. Use the 'payloadObject' utility for extracting the keys
525−-- that should be permitted.
553+-- that should be written.
526--
554--
527−-- There is no built-in mechanism in katip for telling a scribe that
555+-- Scribes provide a finalizer IO action ('scribeFinalizer') that is
528−-- its time to shut down. 'unregisterScribe' merely drops it from the
556+-- meant to synchronously flush any remaining writes and clean up any
529−-- 'LogEnv'. This means there are 2 ways to handle resources as a scribe:
557+-- resources acquired when the scribe was created. Internally, katip
558+-- keeps a buffer for each scribe's writes. When 'closeScribe' or
559+-- 'closeScribes' is called, that buffer stops accepting new log
560+-- messages and after the last item in its buffer is sent to 'liPush',
561+-- calls the finalizer. Thus, when the finalizer returns, katip can
562+-- assume that all resources are cleaned up and all log messages are
563+-- durably written.
530--
564--
531−-- 1. Pass in the resource when the scribe is created. Handle
565+-- While katip internally buffers messages per 'ScribeSettings', it
532−-- allocation and release of the resource elsewhere. This is what the
566+-- sends them one at a time to the scribe. Depending on the scribe
533−-- Handle scribe does.
567+-- itself, it may make sense for that scribe to keep its own internal
534−--
568+-- buffer to batch-send logs if writing items one at a time is not
535−-- 2. Return a finalizing function that tells the scribe to shut
569+-- efficient. The scribe implementer must be sure that on
536−-- down. @katip-elasticsearch@'s @mkEsScribe@ returns an @IO (Scribe,
570+-- finalization, all writes are committed synchronously.
537−-- IO ())@. The finalizer will flush any queued log messages and shut
571+
538−-- down gracefully before returning. This can be hooked into your
572+-- | Signature of a function passed to `Scribe` constructor and
539−-- application's shutdown routine to ensure you never miss any log
573+-- mkScribe* functions that decides which messages to be
540−-- messages on shutdown.
574+-- logged. Typically filters based on 'Severity', but can be
575+-- combined with other, custom logic with 'permitAND' and 'permitOR'
576+type PermitFunc = forall a. Item a -> IO Bool
577+
578+
579+-- | AND together 2 permit functions
580+permitAND :: PermitFunc -> PermitFunc -> PermitFunc
581+permitAND f1 f2 = \a -> liftA2 (&&) (f1 a) (f2 a)
582+
583+-- | OR together 2 permit functions
584+permitOR :: PermitFunc -> PermitFunc -> PermitFunc
585+permitOR f1 f2 = \a -> liftA2 (||) (f1 a) (f2 a)
586+
587+
541data Scribe = Scribe {
588data Scribe = Scribe {
542− liPush :: forall a. LogItem a => Item a -> IO ()
589+ liPush :: forall a. LogItem a => Item a -> IO ()
590+ -- ^ How do we write an item to the scribe's output?
543− , scribeFinalizer :: IO ()
591+ , scribeFinalizer :: IO ()
544− -- ^ Provide a *blocking* finalizer to call when your scribe is
592+ -- ^ Provide a __blocking__ finalizer to call when your scribe is
593+ -- removed. All pending writes should be flushed synchronously. If
545− -- removed. If this is not relevant to your scribe, return () is
594+ -- this is not relevant to your scribe, return () is fine.
546− -- fine.
595+ , scribePermitItem :: PermitFunc
596+ -- ^ Provide a filtering function to allow the item to be logged,
597+ -- or not. It can check Severity or some string in item's
598+ -- body. The initial value of this is usually created from
599+ -- 'permitItem'. Scribes and users can customize this by ANDing
600+ -- or ORing onto the default with 'permitAND' or 'permitOR'
547 }
601 }
548
602
549
603
604+whenM :: Monad m => m Bool -> m () -> m ()
605+whenM mbool = (>>=) mbool . flip when
606+
607+
608+-- | Combine two scribes. Publishes to the left scribe if the left
609+-- would permit the item and to the right scribe if the right would
610+-- permit the item. Finalizers are called in sequence from left to
611+-- right.
550instance Semigroup Scribe where
612instance Semigroup Scribe where
551− (Scribe pushA finA) <> (Scribe pushB finB) =
613+ (Scribe pushA finA permitA) <> (Scribe pushB finB permitB) =
614+ Scribe (\item -> whenM (permitA item) (pushA item)
615+ >> whenM (permitB item) (pushB item)
616+ )
552− Scribe (\item -> pushA item >> pushB item) (finA `finally` finB)
617+ (finA `finally` finB)
618+ (permitOR permitA permitB)
553
619
554
620
555instance Monoid Scribe where
621instance Monoid Scribe where
556− mempty = Scribe (const (return ())) (return ())
622+ mempty = Scribe (const (return ())) (return ()) (permitItem DebugS)
557 mappend = (<>)
623 mappend = (<>)
558
624
559
625
560-------------------------------------------------------------------------------
626-------------------------------------------------------------------------------
561data ScribeHandle = ScribeHandle {
627data ScribeHandle = ScribeHandle {
562 shScribe :: Scribe
628 shScribe :: Scribe
563− , shChan :: BQ.TBQueue WorkerMessage
629+ , shChan :: BQ.TBQueue WorkerMessage
564 }
630 }
565
631
566
632
@@ -572,8 +638,9 @@ data WorkerMessage where
572
638
573-------------------------------------------------------------------------------
639-------------------------------------------------------------------------------
574-- | Should this item be logged given the user's maximum severity?
640-- | Should this item be logged given the user's maximum severity?
641+-- Most new scribes will use this as a base for their 'PermitFunc'
575−permitItem :: Severity -> Item a -> Bool
642+permitItem :: Monad m => Severity -> Item a -> m Bool
576−permitItem sev i = _itemSeverity i >= sev
643+permitItem sev item = return (_itemSeverity item >= sev)
577
644
578
645
579-------------------------------------------------------------------------------
646-------------------------------------------------------------------------------
@@ -598,10 +665,10 @@ makeLenses ''LogEnv
598
665
599
666
600-------------------------------------------------------------------------------
667-------------------------------------------------------------------------------
601−-- | Create a reasonable default InitLogEnv. Uses an 'AutoUdate' with
668+-- | Create a reasonable default InitLogEnv. Uses an 'AutoUpdate' which
602−-- the default settings as the timer. If you are concerned about
669+-- updates the timer every 1ms. If you need even more timestamp
603−-- timestamp precision or event ordering in log outputs like
670+-- precision at the cost of performance, consider setting
604−-- ElasticSearch, you should replace the timer with 'getCurrentTime'
671+-- '_logEnvTimer' with 'getCurrentTime'.
605initLogEnv
672initLogEnv
606 :: Namespace
673 :: Namespace
607 -- ^ A base namespace for this application
674 -- ^ A base namespace for this application
@@ -613,13 +680,15 @@ initLogEnv an env = LogEnv
613 <*> getProcessID
680 <*> getProcessID
614 <*> pure an
681 <*> pure an
615 <*> pure env
682 <*> pure env
616− <*> mkAutoUpdate defaultUpdateSettings { updateAction = getCurrentTime }
683+ <*> mkAutoUpdate defaultUpdateSettings { updateAction = getCurrentTime, updateFreq = 1000 }
617 <*> pure mempty
684 <*> pure mempty
618
685
619
686
620-------------------------------------------------------------------------------
687-------------------------------------------------------------------------------
621-- | Add a scribe to the list. All future log calls will go to this
688-- | Add a scribe to the list. All future log calls will go to this
622−-- scribe in addition to the others.
689+-- scribe in addition to the others. Writes will be buffered per the
690+-- ScribeSettings to prevent slow scribes from slowing down
691+-- logging. Writes will be dropped if the buffer fills.
623registerScribe
692registerScribe
624 :: Text
693 :: Text
625 -- ^ Name the scribe
694 -- ^ Name the scribe
@@ -628,7 +697,7 @@ registerScribe
628 -> LogEnv
697 -> LogEnv
629 -> IO LogEnv
698 -> IO LogEnv
630registerScribe nm scribe ScribeSettings {..} le = do
699registerScribe nm scribe ScribeSettings {..} le = do
631− queue <- atomically (BQ.newTBQueue _scribeBufferSize)
700+ queue <- atomically (BQ.newTBQueue (fromIntegral _scribeBufferSize))
632 worker <- spawnScribeWorker scribe queue
701 worker <- spawnScribeWorker scribe queue
633 let fin = do
702 let fin = do
634 atomically (BQ.writeTBQueue queue PoisonPill)
703 atomically (BQ.writeTBQueue queue PoisonPill)
@@ -643,7 +712,7 @@ registerScribe nm scribe ScribeSettings {..} le = do
643
712
644-------------------------------------------------------------------------------
713-------------------------------------------------------------------------------
645spawnScribeWorker :: Scribe -> BQ.TBQueue WorkerMessage -> IO (Async.Async ())
714spawnScribeWorker :: Scribe -> BQ.TBQueue WorkerMessage -> IO (Async.Async ())
646−spawnScribeWorker (Scribe write _) queue = Async.async go
715+spawnScribeWorker (Scribe write _ _) queue = Async.async go
647 where
716 where
648 go = do
717 go = do
649 newCmd <- atomically (BQ.readTBQueue queue)
718 newCmd <- atomically (BQ.readTBQueue queue)
@@ -672,7 +741,7 @@ defaultScribeSettings = ScribeSettings 4096
672
741
673
742
674-------------------------------------------------------------------------------
743-------------------------------------------------------------------------------
675−-- | Remove a scribe from the environment. This does *not* finalize
744+-- | Remove a scribe from the environment. This does __not__ finalize
676-- the scribe. This mainly only makes sense to use with something like
745-- the scribe. This mainly only makes sense to use with something like
677-- MonadReader's @local@ function to temporarily disavow a single
746-- MonadReader's @local@ function to temporarily disavow a single
678-- logger for a block of code.
747-- logger for a block of code.
@@ -685,7 +754,7 @@ unregisterScribe nm = logEnvScribes %~ M.delete nm
685
754
686
755
687-------------------------------------------------------------------------------
756-------------------------------------------------------------------------------
688−-- | Unregister *all* scribes. Note that this is *not* for closing or
757+-- | Unregister __all__ scribes. Note that this is __not__ for closing or
689-- finalizing scribes, use 'closeScribes' for that. This mainly only
758-- finalizing scribes, use 'closeScribes' for that. This mainly only
690-- makes sense to use with something like MonadReader's @local@
759-- makes sense to use with something like MonadReader's @local@
691-- function to temporarily disavow any loggers for a block of code.
760-- function to temporarily disavow any loggers for a block of code.
@@ -697,9 +766,10 @@ clearScribes = logEnvScribes .~ mempty
697
766
698-------------------------------------------------------------------------------
767-------------------------------------------------------------------------------
699-- | Finalize a scribe. The scribe is removed from the environment,
768-- | Finalize a scribe. The scribe is removed from the environment,
700−-- its finalizer is called and it can never be written to again. Note
769+-- its finalizer is called so that it can never be written to again
770+-- and all pending writes are flushed. Note that this will throw any
701−-- that this will throw any exceptions yoru finalizer will throw, and
771+-- exceptions yoru finalizer will throw, and that LogEnv is immutable,
702−-- that LogEnv is immutable, so it will not be removed in that case.
772+-- so it will not be removed in that case.
703closeScribe
773closeScribe
704 :: Text
774 :: Text
705 -- ^ Name of the scribe
775 -- ^ Name of the scribe
@@ -752,9 +822,11 @@ instance Katip m => Katip (ReaderT s m) where
752 localLogEnv = mapReaderT . localLogEnv
822 localLogEnv = mapReaderT . localLogEnv
753
823
754
824
825+#if !MIN_VERSION_either(4, 5, 0)
755instance Katip m => Katip (EitherT s m) where
826instance Katip m => Katip (EitherT s m) where
756 getLogEnv = lift getLogEnv
827 getLogEnv = lift getLogEnv
757 localLogEnv = mapEitherT . localLogEnv
828 localLogEnv = mapEitherT . localLogEnv
829+#endif
758
830
759
831
760instance Katip m => Katip (ExceptT s m) where
832instance Katip m => Katip (ExceptT s m) where
@@ -808,7 +880,8 @@ instance (Katip m) => Katip (ResourceT m) where
808-- opposed to implementing 'Katip' for your monad.
880-- opposed to implementing 'Katip' for your monad.
809newtype KatipT m a = KatipT { unKatipT :: ReaderT LogEnv m a }
881newtype KatipT m a = KatipT { unKatipT :: ReaderT LogEnv m a }
810 deriving ( Functor, Applicative, Monad, MonadIO
882 deriving ( Functor, Applicative, Monad, MonadIO
811− , MonadMask, MonadCatch, MonadThrow, MonadTrans, MonadBase b)
883+ , MonadMask, MonadCatch, MonadThrow, MonadResource, MonadTrans
884+ , MonadBase b)
812
885
813
886
814instance MonadIO m => Katip (KatipT m) where
887instance MonadIO m => Katip (KatipT m) where
@@ -829,6 +902,21 @@ instance (MonadBaseControl b m) => MonadBaseControl b (KatipT m) where
829 liftBaseWith = defaultLiftBaseWith
902 liftBaseWith = defaultLiftBaseWith
830 restoreM = defaultRestoreM
903 restoreM = defaultRestoreM
831
904
905+instance MonadUnliftIO m => MonadUnliftIO (KatipT m) where
906+#if MIN_VERSION_unliftio_core(0, 2, 0)
907+ withRunInIO inner = KatipT $ ReaderT $ \le -> withRunInIO $ \run ->
908+ inner (run . runKatipT le)
909+#else
910+ askUnliftIO = KatipT $
911+ withUnliftIO $ \u ->
912+ pure (UnliftIO (unliftIO u . unKatipT))
913+#endif
914+
915+#if MIN_VERSION_base(4, 9, 0)
916+instance MF.MonadFail m => MF.MonadFail (KatipT m) where
917+ fail msg = lift (MF.fail msg)
918+ {-# INLINE fail #-}
919+#endif
832
920
833-------------------------------------------------------------------------------
921-------------------------------------------------------------------------------
834-- | Execute 'KatipT' on a log env.
922-- | Execute 'KatipT' on a log env.
@@ -860,21 +948,32 @@ logItem
860 -> m ()
948 -> m ()
861logItem a ns loc sev msg = do
949logItem a ns loc sev msg = do
862 LogEnv{..} <- getLogEnv
950 LogEnv{..} <- getLogEnv
863− liftIO $ do
951+ logKatipItem =<< liftIO
864− item <- Item
865− <$> pure _logEnvApp
952+ (Item <$> pure _logEnvApp
866− <*> pure _logEnvEnv
953+ <*> pure _logEnvEnv
867− <*> pure sev
954+ <*> pure sev
868− <*> (mkThreadIdText <$> myThreadId)
955+ <*> (mkThreadIdText <$> myThreadId)
869− <*> pure _logEnvHost
956+ <*> pure _logEnvHost
870− <*> pure _logEnvPid
957+ <*> pure _logEnvPid
871− <*> pure a
958+ <*> pure a
872− <*> pure msg
959+ <*> pure msg
873− <*> _logEnvTimer
960+ <*> _logEnvTimer
874− <*> pure (_logEnvApp <> ns)
961+ <*> pure (_logEnvApp <> ns)
875− <*> pure loc
962+ <*> pure loc)
963+
964+-- | Log already constructed 'Item'. This is the lowest level function that other log*
965+-- functions use.
966+-- It can be useful when implementing centralised logging services.
967+logKatipItem
968+ :: (A.Applicative m, LogItem a, Katip m)
969+ => Item a
970+ -> m ()
971+logKatipItem item = do
972+ LogEnv{..} <- getLogEnv
973+ liftIO $
974+ FT.forM_ (M.elems _logEnvScribes) $ \ ScribeHandle {..} -> do
975+ whenM (scribePermitItem shScribe item) $
876− FT.forM_ (M.elems _logEnvScribes) $ \ ScribeHandle {..} -> atomically (tryWriteTBQueue shChan (NewItem item))
976+ void $ atomically (tryWriteTBQueue shChan (NewItem item))
877−
878
977
879-------------------------------------------------------------------------------
978-------------------------------------------------------------------------------
880tryWriteTBQueue
979tryWriteTBQueue
@@ -887,6 +986,7 @@ tryWriteTBQueue q a = do
887 unless full (writeTBQueue q a)
986 unless full (writeTBQueue q a)
888 return (not full)
987 return (not full)
889
988
989+
890-------------------------------------------------------------------------------
990-------------------------------------------------------------------------------
891-- | Log with full context, but without any code location.
991-- | Log with full context, but without any code location.
892logF
992logF
@@ -919,7 +1019,7 @@ logException
919logException a ns sev action = action `catchAny` \e -> f e >> throwM e
1019logException a ns sev action = action `catchAny` \e -> f e >> throwM e
920 where
1020 where
921 f e = logF a ns sev (msg e)
1021 f e = logF a ns sev (msg e)
922− msg e = ls (T.pack "An exception has occured: ") <> showLS e
1022+ msg e = ls (T.pack "An exception has occurred: ") <> showLS e
923
1023
924
1024
925-------------------------------------------------------------------------------
1025-------------------------------------------------------------------------------
@@ -973,11 +1073,15 @@ liftLoc (Loc a b c (d1, d2) (e1, e2)) = [|Loc
973-- fill the 'Maybe Loc' gap in 'logF' of this module, and relies on implicit
1073-- fill the 'Maybe Loc' gap in 'logF' of this module, and relies on implicit
974-- callstacks when available (GHC > 7.8).
1074-- callstacks when available (GHC > 7.8).
975#if MIN_VERSION_base(4, 8, 0)
1075#if MIN_VERSION_base(4, 8, 0)
976−getLoc :: (?loc :: CallStack) => Maybe Loc
1076+getLoc :: HasCallStack => Maybe Loc
977−getLoc = case getCallStack ?loc of
1077+getLoc = case getCallStack callStack of
978 [] -> Nothing
1078 [] -> Nothing
979− xs -> Just . toLoc . last $ xs
1079+ xs -> Just . toLoc . head $ filter filterKatip xs
980 where
1080 where
1081+ filterKatip :: (String, SrcLoc) -> Bool
1082+ filterKatip (_, srcloc) = not $
1083+ "katip-" `isPrefixOf` srcLocPackage srcloc
1084+
981 toLoc :: (String, SrcLoc) -> Loc
1085 toLoc :: (String, SrcLoc) -> Loc
982 toLoc (_, l) = Loc {
1086 toLoc (_, l) = Loc {
983 loc_filename = srcLocFile l
1087 loc_filename = srcLocFile l
@@ -1008,7 +1112,7 @@ logT = [| \ a ns sev msg -> logItem a ns (Just $(getLocTH)) sev msg |]
1008
1112
1009
1113
1010-------------------------------------------------------------------------------
1114-------------------------------------------------------------------------------
1011−-- | 'Loc'-tagged logging using implicit-callstacks when available.
1115+-- | 'Loc'-tagged logging using 'GHC.Stack' when available.
1012--
1116--
1013-- This function does not require template-haskell as it
1117-- This function does not require template-haskell as it
1014-- automatically uses <https://hackage.haskell.org/package/base-4.8.2.0/docs/GHC-Stack.html#v:getCallStack implicit-callstacks>
1118-- automatically uses <https://hackage.haskell.org/package/base-4.8.2.0/docs/GHC-Stack.html#v:getCallStack implicit-callstacks>
@@ -1019,7 +1123,7 @@ logT = [| \ a ns sev msg -> logItem a ns (Just $(getLocTH)) sev msg |]
1019--
1123--
1020-- @logLoc obj mempty InfoS "Hello world"@
1124-- @logLoc obj mempty InfoS "Hello world"@
1021#if MIN_VERSION_base(4, 8, 0)
1125#if MIN_VERSION_base(4, 8, 0)
1022−logLoc :: (Applicative m, LogItem a, Katip m, ?loc :: CallStack)
1126+logLoc :: (Applicative m, LogItem a, Katip m, HasCallStack)
1023#else
1127#else
1024logLoc :: (Applicative m, LogItem a, Katip m)
1128logLoc :: (Applicative m, LogItem a, Katip m)
1025#endif
1129#endif
katip/src/Katip/Monadic.hs+114−20
@@ -34,6 +34,8 @@ module Katip.Monadic
34 , katipAddNamespace
34 , katipAddNamespace
35 , katipAddContext
35 , katipAddContext
36 , KatipContextTState(..)
36 , KatipContextTState(..)
37+ , NoLoggingT (..)
38+ , askLoggerIO
37 ) where
39 ) where
38
40
39
41
@@ -42,16 +44,22 @@ import Control.Applicative
42import Control.Exception.Safe
44import Control.Exception.Safe
43import Control.Monad.Base
45import Control.Monad.Base
44import Control.Monad.Error.Class
46import Control.Monad.Error.Class
47+#if MIN_VERSION_base(4, 9, 0)
48+import qualified Control.Monad.Fail as MF
49+#endif
45import Control.Monad.IO.Class
50import Control.Monad.IO.Class
51+import Control.Monad.IO.Unlift
46import Control.Monad.Reader
52import Control.Monad.Reader
47import Control.Monad.State
53import Control.Monad.State
48import Control.Monad.Trans.Control
54import Control.Monad.Trans.Control
55+#if !MIN_VERSION_either(4, 5, 0)
49import Control.Monad.Trans.Either (EitherT, mapEitherT)
56import Control.Monad.Trans.Either (EitherT, mapEitherT)
57+#endif
50import Control.Monad.Trans.Except (ExceptT, mapExceptT)
58import Control.Monad.Trans.Except (ExceptT, mapExceptT)
51import Control.Monad.Trans.Identity (IdentityT, mapIdentityT)
59import Control.Monad.Trans.Identity (IdentityT, mapIdentityT)
52−import Control.Monad.Trans.List (ListT, mapListT)
53import Control.Monad.Trans.Maybe (MaybeT, mapMaybeT)
60import Control.Monad.Trans.Maybe (MaybeT, mapMaybeT)
54−import Control.Monad.Trans.Resource (ResourceT, transResourceT)
61+import Control.Monad.Trans.Resource (MonadResource,
62+ ResourceT, transResourceT)
55import Control.Monad.Trans.RWS (RWST, mapRWST)
63import Control.Monad.Trans.RWS (RWST, mapRWST)
56import qualified Control.Monad.Trans.RWS.Strict as Strict (RWST, mapRWST)
64import qualified Control.Monad.Trans.RWS.Strict as Strict (RWST, mapRWST)
57import qualified Control.Monad.Trans.State.Strict as Strict (StateT, mapStateT)
65import qualified Control.Monad.Trans.State.Strict as Strict (StateT, mapStateT)
@@ -64,6 +72,12 @@ import qualified Data.HashMap.Strict as HM
64import Data.Semigroup as Semi
72import Data.Semigroup as Semi
65import Data.Sequence as Seq
73import Data.Sequence as Seq
66import Data.Text (Text)
74import Data.Text (Text)
75+#if MIN_VERSION_base(4, 8, 0)
76+#if !MIN_VERSION_base(4, 9, 0)
77+import GHC.SrcLoc
78+#endif
79+import GHC.Stack
80+#endif
67import Language.Haskell.TH
81import Language.Haskell.TH
68-------------------------------------------------------------------------------
82-------------------------------------------------------------------------------
69import Katip.Core
83import Katip.Core
@@ -135,7 +149,7 @@ class Katip m => KatipContext m where
135 localKatipContext :: (LogContexts -> LogContexts) -> m a -> m a
149 localKatipContext :: (LogContexts -> LogContexts) -> m a -> m a
136 getKatipNamespace :: m Namespace
150 getKatipNamespace :: m Namespace
137 -- | Temporarily modify the current namespace for the duration of the
151 -- | Temporarily modify the current namespace for the duration of the
138− -- supplied monad. Used in 'katipAddContext'
152+ -- supplied monad. Used in 'katipAddNamespace'
139 localKatipNamespace :: (Namespace -> Namespace) -> m a -> m a
153 localKatipNamespace :: (Namespace -> Namespace) -> m a -> m a
140
154
141instance (KatipContext m, Katip (IdentityT m)) => KatipContext (IdentityT m) where
155instance (KatipContext m, Katip (IdentityT m)) => KatipContext (IdentityT m) where
@@ -152,18 +166,13 @@ instance (KatipContext m, Katip (MaybeT m)) => KatipContext (MaybeT m) where
152 localKatipNamespace = mapMaybeT . localKatipNamespace
166 localKatipNamespace = mapMaybeT . localKatipNamespace
153
167
154
168
169+#if !MIN_VERSION_either(4, 5, 0)
155instance (KatipContext m, Katip (EitherT e m)) => KatipContext (EitherT e m) where
170instance (KatipContext m, Katip (EitherT e m)) => KatipContext (EitherT e m) where
156 getKatipContext = lift getKatipContext
171 getKatipContext = lift getKatipContext
157 localKatipContext = mapEitherT . localKatipContext
172 localKatipContext = mapEitherT . localKatipContext
158 getKatipNamespace = lift getKatipNamespace
173 getKatipNamespace = lift getKatipNamespace
159 localKatipNamespace = mapEitherT . localKatipNamespace
174 localKatipNamespace = mapEitherT . localKatipNamespace
160−
175+#endif
161−
162−instance (KatipContext m, Katip (ListT m)) => KatipContext (ListT m) where
163− getKatipContext = lift getKatipContext
164− localKatipContext = mapListT . localKatipContext
165− getKatipNamespace = lift getKatipNamespace
166− localKatipNamespace = mapListT . localKatipNamespace
167
176
168
177
169instance (KatipContext m, Katip (ReaderT r m)) => KatipContext (ReaderT r m) where
178instance (KatipContext m, Katip (ReaderT r m)) => KatipContext (ReaderT r m) where
@@ -236,7 +245,7 @@ deriving instance (Monad m, KatipContext m) => KatipContext (KatipT m)
236-- very low level and you typically can use 'logTM' in its
245-- very low level and you typically can use 'logTM' in its
237-- place. Automatically supplies payload and namespace.
246-- place. Automatically supplies payload and namespace.
238logItemM
247logItemM
239− :: (Applicative m, KatipContext m)
248+ :: (Applicative m, KatipContext m, HasCallStack)
240 => Maybe Loc
249 => Maybe Loc
241 -> Severity
250 -> Severity
242 -> LogStr
251 -> LogStr
@@ -273,20 +282,21 @@ logTM = [| logItemM (Just $(getLocTH)) |]
273
282
274
283
275-------------------------------------------------------------------------------
284-------------------------------------------------------------------------------
276−-- | 'Loc'-tagged logging when using template-haskell. Automatically
285+-- | 'Loc'-tagged logging when using 'GHC.Stack.getCallStack' implicit-callstacks>.
277−-- supplies payload and namespace.
286+-- Automatically supplies payload and namespace.
278--
287--
279-- Same consideration as `logLoc` applies.
288-- Same consideration as `logLoc` applies.
280--
289--
290+-- Location will be logged from the module that invokes 'logLocM' so
291+-- be aware that wrapping 'logLocM' will make location reporting useless.
292+--
281−-- This function does not require template-haskell as it
293+-- This function does not require template-haskell. Using GHC <= 7.8 will result
282−-- automatically uses <https://hackage.haskell.org/package/base-4.8.2.0/docs/GHC-Stack.html#v:getCallStack implicit-callstacks>
283−-- when the code is compiled using GHC > 7.8. Using an older version of the
284−-- compiler will result in the emission of a log line without any location information,
294+-- in the emission of a log line without any location information.
285−-- so be aware of it. Users using GHC <= 7.8 may want to use the template-haskell function
295+-- Users using GHC <= 7.8 may want to use the template-haskell function
286-- `logTM` for maximum compatibility.
296-- `logTM` for maximum compatibility.
287--
297--
288-- @logLocM InfoS "Hello world"@
298-- @logLocM InfoS "Hello world"@
289−logLocM :: (Applicative m, KatipContext m)
299+logLocM :: (Applicative m, KatipContext m, HasCallStack)
290 => Severity
300 => Severity
291 -> LogStr
301 -> LogStr
292 -> m ()
302 -> m ()
@@ -306,7 +316,7 @@ logExceptionM
306logExceptionM action sev = action `catchAny` \e -> f e >> throwM e
316logExceptionM action sev = action `catchAny` \e -> f e >> throwM e
307 where
317 where
308 f e = logFM sev (msg e)
318 f e = logFM sev (msg e)
309− msg e = ls ("An exception has occured: " :: Text) Semi.<> showLS e
319+ msg e = ls ("An exception has occurred: " :: Text) Semi.<> showLS e
310
320
311
321
312-------------------------------------------------------------------------------
322-------------------------------------------------------------------------------
@@ -340,6 +350,7 @@ newtype KatipContextT m a = KatipContextT {
340 , MonadWriter w
350 , MonadWriter w
341 , MonadError e
351 , MonadError e
342 , MonadPlus
352 , MonadPlus
353+ , MonadResource
343 , Alternative
354 , Alternative
344 , MonadFix
355 , MonadFix
345 , MonadTrans
356 , MonadTrans
@@ -385,6 +396,22 @@ instance (MonadIO m) => KatipContext (KatipContextT m) where
385 getKatipNamespace = KatipContextT $ ReaderT $ \lts -> return (ltsNamespace lts)
396 getKatipNamespace = KatipContextT $ ReaderT $ \lts -> return (ltsNamespace lts)
386 localKatipNamespace f (KatipContextT m) = KatipContextT $ local (\s -> s { ltsNamespace = f (ltsNamespace s)}) m
397 localKatipNamespace f (KatipContextT m) = KatipContextT $ local (\s -> s { ltsNamespace = f (ltsNamespace s)}) m
387
398
399+instance MonadUnliftIO m => MonadUnliftIO (KatipContextT m) where
400+#if MIN_VERSION_unliftio_core(0, 2, 0)
401+ withRunInIO inner = KatipContextT $ ReaderT $ \lts -> withRunInIO $ \run ->
402+ inner (run . runKatipContextT (ltsLogEnv lts) (ltsContext lts) (ltsNamespace lts))
403+#else
404+ askUnliftIO = KatipContextT $
405+ withUnliftIO $ \u ->
406+ pure (UnliftIO (unliftIO u . unKatipContextT))
407+#endif
408+
409+
410+#if MIN_VERSION_base(4, 9, 0)
411+instance MF.MonadFail m => MF.MonadFail (KatipContextT m) where
412+ fail msg = lift (MF.fail msg)
413+ {-# INLINE fail #-}
414+#endif
388
415
389-------------------------------------------------------------------------------
416-------------------------------------------------------------------------------
390runKatipContextT :: (LogItem c) => LogEnv -> c -> Namespace -> KatipContextT m a -> m a
417runKatipContextT :: (LogItem c) => LogEnv -> c -> Namespace -> KatipContextT m a -> m a
@@ -425,3 +452,70 @@ katipAddContext
425 -> m a
452 -> m a
426 -> m a
453 -> m a
427katipAddContext i = localKatipContext (<> (liftPayload i))
454katipAddContext i = localKatipContext (<> (liftPayload i))
455+
456+newtype NoLoggingT m a = NoLoggingT {
457+ runNoLoggingT :: m a
458+ } deriving ( Functor
459+ , Applicative
460+ , Monad
461+ , MonadIO
462+ , MonadThrow
463+ , MonadCatch
464+ , MonadMask
465+ , MonadBase b
466+ , MonadState s
467+ , MonadWriter w
468+ , MonadError e
469+ , MonadPlus
470+ , Alternative
471+ , MonadFix
472+ , MonadReader r
473+ )
474+
475+instance MonadTrans NoLoggingT where
476+ lift = NoLoggingT
477+
478+instance MonadTransControl NoLoggingT where
479+ type StT NoLoggingT a = a
480+ liftWith f = NoLoggingT $ f runNoLoggingT
481+ restoreT = NoLoggingT
482+ {-# INLINE liftWith #-}
483+ {-# INLINE restoreT #-}
484+
485+instance MonadBaseControl b m => MonadBaseControl b (NoLoggingT m) where
486+ type StM (NoLoggingT m) a = StM m a
487+ liftBaseWith f = NoLoggingT $
488+ liftBaseWith $ \runInBase ->
489+ f $ runInBase . runNoLoggingT
490+ restoreM = NoLoggingT . restoreM
491+
492+instance MonadUnliftIO m => MonadUnliftIO (NoLoggingT m) where
493+#if MIN_VERSION_unliftio_core(0, 2, 0)
494+ withRunInIO inner = NoLoggingT $ withRunInIO $ \run ->
495+ inner (run . runNoLoggingT)
496+#else
497+ askUnliftIO = NoLoggingT $
498+ withUnliftIO $ \u ->
499+ pure (UnliftIO (unliftIO u . runNoLoggingT))
500+#endif
501+
502+
503+instance MonadIO m => Katip (NoLoggingT m) where
504+ getLogEnv = liftIO (initLogEnv "NoLoggingT" "no-logging")
505+ localLogEnv = const id
506+
507+instance MonadIO m => KatipContext (NoLoggingT m) where
508+ getKatipContext = pure mempty
509+ localKatipContext = const id
510+ getKatipNamespace = pure mempty
511+ localKatipNamespace = const id
512+
513+
514+-- | Convenience function for when you have to integrate with a third
515+-- party API that takes a generic logging function as an argument.
516+askLoggerIO :: (Applicative m, KatipContext m) => m (Severity -> LogStr -> IO ())
517+askLoggerIO = do
518+ ctx <- getKatipContext
519+ ns <- getKatipNamespace
520+ logEnv <- getLogEnv
521+ pure (\sev msg -> runKatipT logEnv $ logF ctx ns sev msg)
katip/src/Katip/Scribes/Handle.hs+82−31
@@ -1,15 +1,14 @@
1−{-# LANGUAGE RecordWildCards #-}
2−
3module Katip.Scribes.Handle where
1module Katip.Scribes.Handle where
4
2
5-------------------------------------------------------------------------------
3-------------------------------------------------------------------------------
6import Control.Applicative as A
4import Control.Applicative as A
7import Control.Concurrent
5import Control.Concurrent
8import Control.Exception (bracket_, finally)
6import Control.Exception (bracket_, finally)
9−import Control.Monad
10import Data.Aeson
7import Data.Aeson
8+import Data.Text.Lazy (toStrict)
9+import Data.Text.Lazy.Encoding (decodeUtf8)
11import qualified Data.HashMap.Strict as HM
10import qualified Data.HashMap.Strict as HM
12−import Data.Monoid
11+import Data.Monoid as M
13import Data.Scientific as S
12import Data.Scientific as S
14import Data.Text (Text)
13import Data.Text (Text)
15import Data.Text.Lazy.Builder
14import Data.Text.Lazy.Builder
@@ -23,7 +22,7 @@ import Katip.Format.Time (formatAsLogTime)
23
22
24-------------------------------------------------------------------------------
23-------------------------------------------------------------------------------
25brackets :: Builder -> Builder
24brackets :: Builder -> Builder
26−brackets m = fromText "[" <> m <> fromText "]"
25+brackets m = fromText "[" M.<> m <> fromText "]"
27
26
28
27
29-------------------------------------------------------------------------------
28-------------------------------------------------------------------------------
@@ -57,23 +56,36 @@ data ColorStrategy
57-- and other information will be flattened out into bracketed
56-- and other information will be flattened out into bracketed
58-- fields. For example:
57-- fields. For example:
59--
58--
60−-- > [2016-05-11 21:01:15][MyApp][Info][myhost.example.com][1724][ThreadId 1154][main:Helpers.Logging Helpers/Logging.hs:32:7] Started
59+-- > [2016-05-11 21:01:15][MyApp][Info][myhost.example.com][PID 1724][ThreadId 1154][main:Helpers.Logging Helpers/Logging.hs:32:7] Started
61−-- > [2016-05-11 21:01:15][MyApp.confrabulation][Debug][myhost.example.com][1724][ThreadId 1154][confrab_factor:42.0][main:Helpers.Logging Helpers/Logging.hs:41:9] Confrabulating widgets, with extra namespace and context
60+-- > [2016-05-11 21:01:15][MyApp.confrabulation][Debug][myhost.example.com][PID 1724][ThreadId 1154][confrab_factor:42.0][main:Helpers.Logging Helpers/Logging.hs:41:9] Confrabulating widgets, with extra namespace and context
62−-- > [2016-05-11 21:01:15][MyApp][Info][myhost.example.com][1724][ThreadId 1154][main:Helpers.Logging Helpers/Logging.hs:43:7] Namespace and context are back to normal
61+-- > [2016-05-11 21:01:15][MyApp][Info][myhost.example.com][PID 1724][ThreadId 1154][main:Helpers.Logging Helpers/Logging.hs:43:7] Namespace and context are back to normal
63--
62--
64-- Returns the newly-created `Scribe`. The finalizer flushes the
63-- Returns the newly-created `Scribe`. The finalizer flushes the
65-- handle. Handle mode is set to 'LineBuffering' automatically.
64-- handle. Handle mode is set to 'LineBuffering' automatically.
66−mkHandleScribe :: ColorStrategy -> Handle -> Severity -> Verbosity -> IO Scribe
65+mkHandleScribe :: ColorStrategy -> Handle -> PermitFunc -> Verbosity -> IO Scribe
67−mkHandleScribe cs h sev verb = do
66+mkHandleScribe = mkHandleScribeWithFormatter bracketFormat
67+
68+-- | Logs to a file handle such as stdout, stderr, or a file. Takes a custom
69+-- `ItemFormatter` that can be used to format `Item` as needed.
70+--
71+-- Returns the newly-created `Scribe`. The finalizer flushes the
72+-- handle. Handle mode is set to 'LineBuffering' automatically.
73+mkHandleScribeWithFormatter :: (forall a . LogItem a => ItemFormatter a)
74+ -> ColorStrategy
75+ -> Handle
76+ -> PermitFunc
77+ -> Verbosity
78+ -> IO Scribe
79+mkHandleScribeWithFormatter itemFormatter cs h permitF verb = do
68 hSetBuffering h LineBuffering
80 hSetBuffering h LineBuffering
69 colorize <- case cs of
81 colorize <- case cs of
70 ColorIfTerminal -> hIsTerminalDevice h
82 ColorIfTerminal -> hIsTerminalDevice h
71 ColorLog b -> return b
83 ColorLog b -> return b
72 lock <- newMVar ()
84 lock <- newMVar ()
73 let logger i@Item{..} = do
85 let logger i@Item{..} = do
74− when (permitItem sev i) $ bracket_ (takeMVar lock) (putMVar lock ()) $
86+ bracket_ (takeMVar lock) (putMVar lock ()) $
75− T.hPutStrLn h $ toLazyText $ formatItem colorize verb i
87+ T.hPutStrLn h $ toLazyText $ itemFormatter colorize verb i
76− return $ Scribe logger (hFlush h)
88+ return $ Scribe logger (hFlush h) permitF
77
89
78
90
79-------------------------------------------------------------------------------
91-------------------------------------------------------------------------------
@@ -82,35 +94,74 @@ mkHandleScribe cs h sev verb = do
82-- 'AppendMode' and will close the file handle on
94-- 'AppendMode' and will close the file handle on
83-- 'closeScribe'/'closeScribes'. Does not do log coloring. Sets handle
95-- 'closeScribe'/'closeScribes'. Does not do log coloring. Sets handle
84-- to 'LineBuffering' mode.
96-- to 'LineBuffering' mode.
85−mkFileScribe :: FilePath -> Severity -> Verbosity -> IO Scribe
97+mkFileScribe :: FilePath -> PermitFunc -> Verbosity -> IO Scribe
86−mkFileScribe f sev verb = do
98+mkFileScribe f permitF verb = do
87 h <- openFile f AppendMode
99 h <- openFile f AppendMode
88− Scribe logger finalizer <- mkHandleScribe (ColorLog False) h sev verb
100+ Scribe logger finalizer permit <- mkHandleScribe (ColorLog False) h permitF verb
89− return (Scribe logger (finalizer `finally` hClose h))
101+ return (Scribe logger (finalizer `finally` hClose h) permit)
90
102
91
103
92-------------------------------------------------------------------------------
104-------------------------------------------------------------------------------
105+-- | A custom ItemFormatter for logging `Item`s. Takes a `Bool` indicating
106+-- whether to colorize the output, `Verbosity` of output, and an `Item` to
107+-- format.
108+--
109+-- See `bracketFormat` and `jsonFormat` for examples.
93−formatItem :: LogItem a => Bool -> Verbosity -> Item a -> Builder
110+type ItemFormatter a = Bool -> Verbosity -> Item a -> Builder
111+
112+
113+formatItem :: LogItem a => ItemFormatter a
114+formatItem = bracketFormat
115+{-# DEPRECATED formatItem "Use bracketFormat instead" #-}
116+
117+-- | A traditional 'bracketed' log format. Contexts and other information will
118+-- be flattened out into bracketed fields. For example:
119+--
120+-- > [2016-05-11 21:01:15][MyApp][Info][myhost.example.com][PID 1724][ThreadId 1154][main:Helpers.Logging Helpers/Logging.hs:32:7] Started
121+-- > [2016-05-11 21:01:15][MyApp.confrabulation][Debug][myhost.example.com][PID 1724][ThreadId 1154][confrab_factor:42.0][main:Helpers.Logging Helpers/Logging.hs:41:9] Confrabulating widgets, with extra namespace and context
122+-- > [2016-05-11 21:01:15][MyApp][Info][myhost.example.com][PID 1724][ThreadId 1154][main:Helpers.Logging Helpers/Logging.hs:43:7] Namespace and context are back to normal
123+bracketFormat :: LogItem a => ItemFormatter a
94−formatItem withColor verb Item{..} =
124+bracketFormat withColor verb Item{..} =
95 brackets nowStr <>
125 brackets nowStr <>
96 brackets (mconcat $ map fromText $ intercalateNs _itemNamespace) <>
126 brackets (mconcat $ map fromText $ intercalateNs _itemNamespace) <>
97 brackets (fromText (renderSeverity' _itemSeverity)) <>
127 brackets (fromText (renderSeverity' _itemSeverity)) <>
98 brackets (fromString _itemHost) <>
128 brackets (fromString _itemHost) <>
99− brackets (fromString (show _itemProcess)) <>
129+ brackets ("PID " <> fromString (show _itemProcess)) <>
100− brackets (fromText (getThreadIdText _itemThread)) <>
130+ brackets ("ThreadId " <> fromText (getThreadIdText _itemThread)) <>
101 mconcat ks <>
131 mconcat ks <>
102 maybe mempty (brackets . fromString . locationToString) _itemLoc <>
132 maybe mempty (brackets . fromString . locationToString) _itemLoc <>
103 fromText " " <> (unLogStr _itemMessage)
133 fromText " " <> (unLogStr _itemMessage)
104 where
134 where
105 nowStr = fromText (formatAsLogTime _itemTime)
135 nowStr = fromText (formatAsLogTime _itemTime)
106 ks = map brackets $ getKeys verb _itemPayload
136 ks = map brackets $ getKeys verb _itemPayload
107− renderSeverity' s = case s of
137+ renderSeverity' severity =
138+ colorBySeverity withColor severity (renderSeverity severity)
139+
140+-- | Logs items as JSON. This can be useful in circumstances where you already
141+-- have infrastructure that is expecting JSON to be logged to a standard stream
142+-- or file. For example:
143+--
144+-- > {"at":"2018-10-02T21:50:30.4523848Z","env":"production","ns":["MyApp"],"data":{},"app":["MyApp"],"msg":"Started","pid":"10456","loc":{"loc_col":9,"loc_pkg":"main","loc_mod":"Helpers.Logging","loc_fn":"Helpers\\Logging.hs","loc_ln":44},"host":"myhost.example.com","sev":"Info","thread":"ThreadId 139"}
145+-- > {"at":"2018-10-02T21:50:30.4523848Z","env":"production","ns":["MyApp","confrabulation"],"data":{"confrab_factor":42},"app":["MyApp"],"msg":"Confrabulating widgets, with extra namespace and context","pid":"10456","loc":{"loc_col":11,"loc_pkg":"main","loc_mod":"Helpers.Logging","loc_fn":"Helpers\\Logging.hs","loc_ln":53},"host":"myhost.example.com","sev":"Debug","thread":"ThreadId 139"}
146+-- > {"at":"2018-10-02T21:50:30.4523848Z","env":"production","ns":["MyApp"],"data":{},"app":["MyApp"],"msg":"Namespace and context are back to normal","pid":"10456","loc":{"loc_col":9,"loc_pkg":"main","loc_mod":"Helpers.Logging","loc_fn":"Helpers\\Logging.hs","loc_ln":55},"host":"myhost.example.com","sev":"Info","thread":"ThreadId 139"}
147+jsonFormat :: LogItem a => ItemFormatter a
148+jsonFormat withColor verb i =
149+ fromText $
150+ colorBySeverity withColor (_itemSeverity i) $
151+ toStrict $ decodeUtf8 $ encode $ itemJson verb i
152+
153+-- | Color a text message based on `Severity`. `ErrorS` and more severe errors
154+-- are colored red, `WarningS` is colored yellow, and all other messages are
155+-- rendered in the default color.
156+colorBySeverity :: Bool -> Severity -> Text -> Text
157+colorBySeverity withColor severity msg = case severity of
108− EmergencyS -> red $ renderSeverity s
158+ EmergencyS -> red msg
109− AlertS -> red $ renderSeverity s
159+ AlertS -> red msg
110− CriticalS -> red $ renderSeverity s
160+ CriticalS -> red msg
111− ErrorS -> red $ renderSeverity s
161+ ErrorS -> red msg
112− WarningS -> yellow $ renderSeverity s
162+ WarningS -> yellow msg
113− _ -> renderSeverity s
163+ _ -> msg
164+ where
114 red = colorize "31"
165 red = colorize "31"
115 yellow = colorize "33"
166 yellow = colorize "33"
116 colorize c s
167 colorize c s
@@ -122,8 +173,8 @@ formatItem withColor verb Item{..} =
122-- stdout. This is a decent example of how to build a LogEnv and is
173-- stdout. This is a decent example of how to build a LogEnv and is
123-- best for scripts that just need a quick, reasonable set up to log
174-- best for scripts that just need a quick, reasonable set up to log
124-- to stdout.
175-- to stdout.
125−ioLogEnv :: Severity -> Verbosity -> IO LogEnv
176+ioLogEnv :: PermitFunc -> Verbosity -> IO LogEnv
126−ioLogEnv sev verb = do
177+ioLogEnv permit verb = do
127 le <- initLogEnv "io" "io"
178 le <- initLogEnv "io" "io"
128− lh <- mkHandleScribe ColorIfTerminal stdout sev verb
179+ lh <- mkHandleScribe ColorIfTerminal stdout permit verb
129 registerScribe "stdout" lh defaultScribeSettings le
180 registerScribe "stdout" lh defaultScribeSettings le
katip/test/Katip/Tests.hs+18−4
@@ -19,7 +19,7 @@ import Control.Exception.Safe
19import Data.Aeson
19import Data.Aeson
20import qualified Data.HashMap.Strict as HM
20import qualified Data.HashMap.Strict as HM
21import qualified Data.Map.Strict as M
21import qualified Data.Map.Strict as M
22−import Data.Monoid
22+import Data.Monoid as Monoid
23import Data.Text (Text)
23import Data.Text (Text)
24import qualified Data.Text.Lazy.Builder as B
24import qualified Data.Text.Lazy.Builder as B
25import Data.Time
25import Data.Time
@@ -42,6 +42,8 @@ tests = testGroup "Katip"
42 [
42 [
43 testProperty "JSON cycle Item" $ \(i :: Item ()) ->
43 testProperty "JSON cycle Item" $ \(i :: Item ()) ->
44 prop_json_cycle i
44 prop_json_cycle i
45+ , testProperty "JSON cycle verbosity" $ \(v :: Verbosity) ->
46+ prop_json_cycle v
45 , eqItemTests
47 , eqItemTests
46 , testProperty "renderSeverity/textToSeverity cycle" $ \sev ->
48 , testProperty "renderSeverity/textToSeverity cycle" $ \sev ->
47 textToSeverity(renderSeverity sev) === Just sev
49 textToSeverity(renderSeverity sev) === Just sev
@@ -124,14 +126,22 @@ loggingTests = testGroup "logging"
124 , (baseNS <> "added" <> "namespace", baseCtx, "with namespaces")
126 , (baseNS <> "added" <> "namespace", baseCtx, "with namespaces")
125 , (baseNS <> "added", HM.insert "additional" (Bool True) baseCtx, "additional context")
127 , (baseNS <> "added", HM.insert "additional" (Bool True) baseCtx, "additional context")
126 ]
128 ]
129+ , testCase "Katip.Monadic.logLocM" $ do
130+ (le, items) <- recordingEnv
131+ runKatipContextT le (sl "base_context" (42 :: Int)) "base_namespace" $ logLocM InfoS "basic log"
132+ _ <- closeScribes le
133+ loggedItems <- readTVarIO items
134+ let loc = _itemLoc (head loggedItems)
135+ fmap loc_module loc @?= Just "Katip.Tests"
127 ]
136 ]
128 where
137 where
129 recordingEnv :: IO (LogEnv, TVar [Item Object])
138 recordingEnv :: IO (LogEnv, TVar [Item Object])
130 recordingEnv = do
139 recordingEnv = do
131− items <- newTVarIO mempty
140+ items <- newTVarIO Monoid.mempty
132 let scribe = Scribe
141 let scribe = Scribe
133 { liPush = \i -> atomically (modifyTVar' items (<> [toObject <$> i]))
142 { liPush = \i -> atomically (modifyTVar' items (<> [toObject <$> i]))
134 , scribeFinalizer = return ()
143 , scribeFinalizer = return ()
144+ , scribePermitItem = permitItem DebugS
135 }
145 }
136 le1 <- initLogEnv "tests" "test"
146 le1 <- initLogEnv "tests" "test"
137 le2 <- registerScribe "recorder" scribe defaultScribeSettings le1
147 le2 <- registerScribe "recorder" scribe defaultScribeSettings le1
@@ -145,7 +155,7 @@ trivialScribe :: IO (Scribe, TVar Bool)
145trivialScribe = do
155trivialScribe = do
146 finalizerCalled <- newTVarIO False
156 finalizerCalled <- newTVarIO False
147 let finalizer = atomically (writeTVar finalizerCalled True)
157 let finalizer = atomically (writeTVar finalizerCalled True)
148− return (Scribe (const (return ())) finalizer, finalizerCalled)
158+ return (Scribe (const (return ())) finalizer (permitItem DebugS), finalizerCalled)
149
159
150
160
151-------------------------------------------------------------------------------
161-------------------------------------------------------------------------------
@@ -155,7 +165,7 @@ brokenScribe scribeNum = do
155 let finalizer = do
165 let finalizer = do
156 atomically (writeTVar finalizerCalled True)
166 atomically (writeTVar finalizerCalled True)
157 throw (ScribeBroken scribeNum)
167 throw (ScribeBroken scribeNum)
158− return (Scribe (const (return ())) finalizer, finalizerCalled)
168+ return (Scribe (const (return ())) finalizer (permitItem DebugS), finalizerCalled)
159
169
160
170
161-------------------------------------------------------------------------------
171-------------------------------------------------------------------------------
@@ -259,6 +269,10 @@ instance Arbitrary Loc where
259 return $ Loc f p m s s
269 return $ Loc f p m s s
260
270
261
271
272+-------------------------------------------------------------------------------
273+instance Arbitrary Verbosity where
274+ arbitrary = oneof $ map pure [V0, V1, V2, V3]
275+
262-------------------------------------------------------------------------------
276-------------------------------------------------------------------------------
263instance Arbitrary Severity where
277instance Arbitrary Severity where
264 arbitrary = oneof $ map pure [ DebugS
278 arbitrary = oneof $ map pure [ DebugS
katip/test/Katip/Tests/Scribes/Handle-json.goldennew file+38
@@ -0,0 +1,38 @@
1+{"at":"2016-06-12T12:34:56Z","env":"production","ns":["foo"],"data":{"note":{"deep":"some note"}},"app":["app"],"msg":"message","pid":"7331","loc":null,"host":"example","sev":"Debug","thread":"1337"}
2+{"at":"2016-06-12T12:34:56Z","env":"production","ns":["foo"],"data":{"note":{"deep":"some note"}},"app":["app"],"msg":"message","pid":"7331","loc":null,"host":"example","sev":"Info","thread":"1337"}
3+{"at":"2016-06-12T12:34:56Z","env":"production","ns":["foo"],"data":{"note":{"deep":"some note"}},"app":["app"],"msg":"message","pid":"7331","loc":null,"host":"example","sev":"Notice","thread":"1337"}
4+{"at":"2016-06-12T12:34:56Z","env":"production","ns":["foo"],"data":{"note":{"deep":"some note"}},"app":["app"],"msg":"message","pid":"7331","loc":null,"host":"example","sev":"Warning","thread":"1337"}
5+{"at":"2016-06-12T12:34:56Z","env":"production","ns":["foo"],"data":{"note":{"deep":"some note"}},"app":["app"],"msg":"message","pid":"7331","loc":null,"host":"example","sev":"Error","thread":"1337"}
6+{"at":"2016-06-12T12:34:56Z","env":"production","ns":["foo"],"data":{"note":{"deep":"some note"}},"app":["app"],"msg":"message","pid":"7331","loc":null,"host":"example","sev":"Critical","thread":"1337"}
7+{"at":"2016-06-12T12:34:56Z","env":"production","ns":["foo"],"data":{"note":{"deep":"some note"}},"app":["app"],"msg":"message","pid":"7331","loc":null,"host":"example","sev":"Alert","thread":"1337"}
8+{"at":"2016-06-12T12:34:56Z","env":"production","ns":["foo"],"data":{"note":{"deep":"some note"}},"app":["app"],"msg":"message","pid":"7331","loc":null,"host":"example","sev":"Emergency","thread":"1337"}
9+{"at":"2016-06-12T12:34:56Z","env":"production","ns":["foo"],"data":{"note":{"deep":"some note"}},"app":["app"],"msg":"message","pid":"7331","loc":null,"host":"example","sev":"Info","thread":"0"}
10+{"at":"2016-06-12T12:34:56Z","env":"production","ns":["foo"],"data":{"note":{"deep":"some note"}},"app":["app"],"msg":"message","pid":"7331","loc":null,"host":"example","sev":"Info","thread":"1"}
11+{"at":"2016-06-12T12:34:56Z","env":"production","ns":["foo"],"data":{"note":{"deep":"some note"}},"app":["app"],"msg":"message","pid":"7331","loc":null,"host":"example","sev":"Info","thread":"1337"}
12+{"at":"2016-06-12T12:34:56Z","env":"production","ns":["foo"],"data":{"note":{"deep":"some note"}},"app":["app"],"msg":"message","pid":"7331","loc":null,"host":"example","sev":"Info","thread":"2147483647"}
13+{"at":"2016-06-12T12:34:56Z","env":"production","ns":["foo"],"data":{"note":{"deep":"some note"}},"app":["app"],"msg":"message","pid":"7331","loc":null,"host":"example","sev":"Info","thread":"1337"}
14+{"at":"2016-06-12T12:34:56Z","env":"production","ns":["foo"],"data":{"note":{"deep":"some note"}},"app":["app"],"msg":"message","pid":"7331","loc":null,"host":"www.example.com","sev":"Info","thread":"1337"}
15+{"at":"2016-06-12T12:34:56Z","env":"production","ns":["foo"],"data":{"note":{"deep":"some note"}},"app":["app"],"msg":"message","pid":"7331","loc":null,"host":"127.0.0.1","sev":"Info","thread":"1337"}
16+{"at":"2016-06-12T12:34:56Z","env":"production","ns":["foo"],"data":{"note":{"deep":"some note"}},"app":["app"],"msg":"message","pid":"0","loc":null,"host":"example","sev":"Info","thread":"1337"}
17+{"at":"2016-06-12T12:34:56Z","env":"production","ns":["foo"],"data":{"note":{"deep":"some note"}},"app":["app"],"msg":"message","pid":"1","loc":null,"host":"example","sev":"Info","thread":"1337"}
18+{"at":"2016-06-12T12:34:56Z","env":"production","ns":["foo"],"data":{"note":{"deep":"some note"}},"app":["app"],"msg":"message","pid":"1337","loc":null,"host":"example","sev":"Info","thread":"1337"}
19+{"at":"2016-06-12T12:34:56Z","env":"production","ns":["foo"],"data":{"note":{"deep":"some note"}},"app":["app"],"msg":"message","pid":"2147483647","loc":null,"host":"example","sev":"Info","thread":"1337"}
20+{"at":"2016-06-12T12:34:56Z","env":"production","ns":["foo"],"data":{"note":{"deep":"some note"}},"app":["app"],"msg":"message","pid":"7331","loc":null,"host":"example","sev":"Info","thread":"1337"}
21+{"at":"2016-06-12T12:34:56Z","env":"production","ns":["foo"],"data":{"note":{"deep":"some note"}},"app":["app"],"msg":"message\nwith newline","pid":"7331","loc":null,"host":"example","sev":"Info","thread":"1337"}
22+{"at":"2016-06-12T12:34:56Z","env":"production","ns":["foo"],"data":{"note":{"deep":"some note"}},"app":["app"],"msg":" a really long message a really long message a really long message a really long message a really long message a really long message a really long message a really long message a really long message a really long message a really long message a really long message a really long message a really long message a really long message a really long message a really long message a really long message a really long message a really long message a really long message a really long message a really long message a really long message a really long message a really long message a really long message a really long message a really long message a really long message a really long message a really long message a really long message a really long message a really long message a really long message a really long message a really long message a really long message a really long message","pid":"7331","loc":null,"host":"example","sev":"Info","thread":"1337"}
23+{"at":"2016-06-12T12:34:56Z","env":"production","ns":["foo"],"data":{"note":{"deep":"some note"}},"app":["app"],"msg":"сообщение","pid":"7331","loc":null,"host":"example","sev":"Info","thread":"1337"}
24+{"at":"2016-06-12T12:34:56Z","env":"production","ns":["foo"],"data":{"note":{"deep":"some note"}},"app":["app"],"msg":"哈囉世界","pid":"7331","loc":null,"host":"example","sev":"Info","thread":"1337"}
25+{"at":"2000-01-01T00:00:00Z","env":"production","ns":["foo"],"data":{"note":{"deep":"some note"}},"app":["app"],"msg":"message","pid":"7331","loc":null,"host":"example","sev":"Info","thread":"1337"}
26+{"at":"2123-12-31T23:59:59.999999999999Z","env":"production","ns":["foo"],"data":{"note":{"deep":"some note"}},"app":["app"],"msg":"message","pid":"7331","loc":null,"host":"example","sev":"Info","thread":"1337"}
27+{"at":"2016-10-10T01:01:05Z","env":"production","ns":["foo"],"data":{"note":{"deep":"some note"}},"app":["app"],"msg":"message","pid":"7331","loc":null,"host":"example","sev":"Info","thread":"1337"}
28+{"at":"2100-12-31T12:59:10.1Z","env":"production","ns":["foo"],"data":{"note":{"deep":"some note"}},"app":["app"],"msg":"message","pid":"7331","loc":null,"host":"example","sev":"Info","thread":"1337"}
29+{"at":"1982-01-01T12:30:00.000000000001Z","env":"production","ns":["foo"],"data":{"note":{"deep":"some note"}},"app":["app"],"msg":"message","pid":"7331","loc":null,"host":"example","sev":"Info","thread":"1337"}
30+{"at":"2016-06-12T12:34:56Z","env":"production","ns":["foo"],"data":{"note":{"deep":"some note"}},"app":["app"],"msg":"message","pid":"7331","loc":null,"host":"example","sev":"Info","thread":"1337"}
31+{"at":"2016-06-12T12:34:56Z","env":"production","ns":["foo","bar"],"data":{"note":{"deep":"some note"}},"app":["app"],"msg":"message","pid":"7331","loc":null,"host":"example","sev":"Info","thread":"1337"}
32+{"at":"2016-06-12T12:34:56Z","env":"production","ns":["фу","бар"],"data":{"note":{"deep":"some note"}},"app":["app"],"msg":"message","pid":"7331","loc":null,"host":"example","sev":"Info","thread":"1337"}
33+{"at":"2016-06-12T12:34:56Z","env":"production","ns":["with\nnewline"],"data":{"note":{"deep":"some note"}},"app":["app"],"msg":"message","pid":"7331","loc":null,"host":"example","sev":"Info","thread":"1337"}
34+{"at":"2016-06-12T12:34:56Z","env":"production","ns":["foo"],"data":{"note":{"deep":"some note"}},"app":["app"],"msg":"message","pid":"7331","loc":null,"host":"example","sev":"Info","thread":"1337"}
35+{"at":"2016-06-12T12:34:56Z","env":"production","ns":["foo"],"data":{"note":{"deep":"some note"}},"app":["app"],"msg":"message","pid":"7331","loc":{"loc_col":1,"loc_pkg":"main","loc_mod":"Some.Module","loc_fn":"path/Some/Module.hs","loc_ln":30},"host":"example","sev":"Info","thread":"1337"}
36+{"at":"2016-06-12T12:34:56Z","env":"production","ns":["foo"],"data":{"note":{"deep":"some note"}},"app":["app"],"msg":"message","pid":"7331","loc":{"loc_col":9000,"loc_pkg":"main","loc_mod":"Some.Module","loc_fn":"путь/Some/Module.hs","loc_ln":3000},"host":"example","sev":"Info","thread":"1337"}
37+{"at":"2016-06-12T12:34:56Z","env":"production","ns":["foo"],"data":{"sub":null,"list":[],"text":"","num":0,"float":0},"app":["app"],"msg":"message","pid":"7331","loc":null,"host":"example","sev":"Info","thread":"1337"}
38+{"at":"2016-06-12T12:34:56Z","env":"production","ns":["foo"],"data":{"sub":{"note":{"deep":"some note"}},"list":["one","two","three"],"text":"note","num":10,"float":5.5},"app":["app"],"msg":"message","pid":"7331","loc":null,"host":"example","sev":"Info","thread":"1337"}
katip/test/Katip/Tests/Scribes/Handle-text.golden+38−38
@@ -1,40 +1,40 @@
1−[2016-06-12 12:34:56][foo][Debug][example][7331][1337][note.deep:some note] message
1+[2016-06-12 12:34:56][foo][Debug][example][PID 7331][ThreadId 1337][note.deep:some note] message
2−[2016-06-12 12:34:56][foo][Info][example][7331][1337][note.deep:some note] message
2+[2016-06-12 12:34:56][foo][Info][example][PID 7331][ThreadId 1337][note.deep:some note] message
3−[2016-06-12 12:34:56][foo][Notice][example][7331][1337][note.deep:some note] message
3+[2016-06-12 12:34:56][foo][Notice][example][PID 7331][ThreadId 1337][note.deep:some note] message
4−[2016-06-12 12:34:56][foo][Warning][example][7331][1337][note.deep:some note] message
4+[2016-06-12 12:34:56][foo][Warning][example][PID 7331][ThreadId 1337][note.deep:some note] message
5−[2016-06-12 12:34:56][foo][Error][example][7331][1337][note.deep:some note] message
5+[2016-06-12 12:34:56][foo][Error][example][PID 7331][ThreadId 1337][note.deep:some note] message
6−[2016-06-12 12:34:56][foo][Critical][example][7331][1337][note.deep:some note] message
6+[2016-06-12 12:34:56][foo][Critical][example][PID 7331][ThreadId 1337][note.deep:some note] message
7−[2016-06-12 12:34:56][foo][Alert][example][7331][1337][note.deep:some note] message
7+[2016-06-12 12:34:56][foo][Alert][example][PID 7331][ThreadId 1337][note.deep:some note] message
8−[2016-06-12 12:34:56][foo][Emergency][example][7331][1337][note.deep:some note] message
8+[2016-06-12 12:34:56][foo][Emergency][example][PID 7331][ThreadId 1337][note.deep:some note] message
9−[2016-06-12 12:34:56][foo][Info][example][7331][0][note.deep:some note] message
9+[2016-06-12 12:34:56][foo][Info][example][PID 7331][ThreadId 0][note.deep:some note] message
10−[2016-06-12 12:34:56][foo][Info][example][7331][1][note.deep:some note] message
10+[2016-06-12 12:34:56][foo][Info][example][PID 7331][ThreadId 1][note.deep:some note] message
11−[2016-06-12 12:34:56][foo][Info][example][7331][1337][note.deep:some note] message
11+[2016-06-12 12:34:56][foo][Info][example][PID 7331][ThreadId 1337][note.deep:some note] message
12−[2016-06-12 12:34:56][foo][Info][example][7331][2147483647][note.deep:some note] message
12+[2016-06-12 12:34:56][foo][Info][example][PID 7331][ThreadId 2147483647][note.deep:some note] message
13−[2016-06-12 12:34:56][foo][Info][example][7331][1337][note.deep:some note] message
13+[2016-06-12 12:34:56][foo][Info][example][PID 7331][ThreadId 1337][note.deep:some note] message
14−[2016-06-12 12:34:56][foo][Info][www.example.com][7331][1337][note.deep:some note] message
14+[2016-06-12 12:34:56][foo][Info][www.example.com][PID 7331][ThreadId 1337][note.deep:some note] message
15−[2016-06-12 12:34:56][foo][Info][127.0.0.1][7331][1337][note.deep:some note] message
15+[2016-06-12 12:34:56][foo][Info][127.0.0.1][PID 7331][ThreadId 1337][note.deep:some note] message
16−[2016-06-12 12:34:56][foo][Info][example][0][1337][note.deep:some note] message
16+[2016-06-12 12:34:56][foo][Info][example][PID 0][ThreadId 1337][note.deep:some note] message
17−[2016-06-12 12:34:56][foo][Info][example][1][1337][note.deep:some note] message
17+[2016-06-12 12:34:56][foo][Info][example][PID 1][ThreadId 1337][note.deep:some note] message
18−[2016-06-12 12:34:56][foo][Info][example][1337][1337][note.deep:some note] message
18+[2016-06-12 12:34:56][foo][Info][example][PID 1337][ThreadId 1337][note.deep:some note] message
19−[2016-06-12 12:34:56][foo][Info][example][2147483647][1337][note.deep:some note] message
19+[2016-06-12 12:34:56][foo][Info][example][PID 2147483647][ThreadId 1337][note.deep:some note] message
20−[2016-06-12 12:34:56][foo][Info][example][7331][1337][note.deep:some note] message
20+[2016-06-12 12:34:56][foo][Info][example][PID 7331][ThreadId 1337][note.deep:some note] message
21−[2016-06-12 12:34:56][foo][Info][example][7331][1337][note.deep:some note] message
21+[2016-06-12 12:34:56][foo][Info][example][PID 7331][ThreadId 1337][note.deep:some note] message
22with newline
22with newline
23−[2016-06-12 12:34:56][foo][Info][example][7331][1337][note.deep:some note] a really long message a really long message a really long message a really long message a really long message a really long message a really long message a really long message a really long message a really long message a really long message a really long message a really long message a really long message a really long message a really long message a really long message a really long message a really long message a really long message a really long message a really long message a really long message a really long message a really long message a really long message a really long message a really long message a really long message a really long message a really long message a really long message a really long message a really long message a really long message a really long message a really long message a really long message a really long message a really long message
23+[2016-06-12 12:34:56][foo][Info][example][PID 7331][ThreadId 1337][note.deep:some note] a really long message a really long message a really long message a really long message a really long message a really long message a really long message a really long message a really long message a really long message a really long message a really long message a really long message a really long message a really long message a really long message a really long message a really long message a really long message a really long message a really long message a really long message a really long message a really long message a really long message a really long message a really long message a really long message a really long message a really long message a really long message a really long message a really long message a really long message a really long message a really long message a really long message a really long message a really long message a really long message
24−[2016-06-12 12:34:56][foo][Info][example][7331][1337][note.deep:some note] сообщение
24+[2016-06-12 12:34:56][foo][Info][example][PID 7331][ThreadId 1337][note.deep:some note] сообщение
25−[2016-06-12 12:34:56][foo][Info][example][7331][1337][note.deep:some note] 哈囉世界
25+[2016-06-12 12:34:56][foo][Info][example][PID 7331][ThreadId 1337][note.deep:some note] 哈囉世界
26−[2000-01-01 00:00:00][foo][Info][example][7331][1337][note.deep:some note] message
26+[2000-01-01 00:00:00][foo][Info][example][PID 7331][ThreadId 1337][note.deep:some note] message
27−[2123-12-31 23:59:59][foo][Info][example][7331][1337][note.deep:some note] message
27+[2123-12-31 23:59:59][foo][Info][example][PID 7331][ThreadId 1337][note.deep:some note] message
28−[2016-10-10 01:01:05][foo][Info][example][7331][1337][note.deep:some note] message
28+[2016-10-10 01:01:05][foo][Info][example][PID 7331][ThreadId 1337][note.deep:some note] message
29−[2100-12-31 12:59:10][foo][Info][example][7331][1337][note.deep:some note] message
29+[2100-12-31 12:59:10][foo][Info][example][PID 7331][ThreadId 1337][note.deep:some note] message
30−[1982-01-01 12:30:00][foo][Info][example][7331][1337][note.deep:some note] message
30+[1982-01-01 12:30:00][foo][Info][example][PID 7331][ThreadId 1337][note.deep:some note] message
31−[2016-06-12 12:34:56][foo][Info][example][7331][1337][note.deep:some note] message
31+[2016-06-12 12:34:56][foo][Info][example][PID 7331][ThreadId 1337][note.deep:some note] message
32−[2016-06-12 12:34:56][foo.bar][Info][example][7331][1337][note.deep:some note] message
32+[2016-06-12 12:34:56][foo.bar][Info][example][PID 7331][ThreadId 1337][note.deep:some note] message
33−[2016-06-12 12:34:56][фу.бар][Info][example][7331][1337][note.deep:some note] message
33+[2016-06-12 12:34:56][фу.бар][Info][example][PID 7331][ThreadId 1337][note.deep:some note] message
34[2016-06-12 12:34:56][with
34[2016-06-12 12:34:56][with
35−newline][Info][example][7331][1337][note.deep:some note] message
35+newline][Info][example][PID 7331][ThreadId 1337][note.deep:some note] message
36−[2016-06-12 12:34:56][foo][Info][example][7331][1337][note.deep:some note] message
36+[2016-06-12 12:34:56][foo][Info][example][PID 7331][ThreadId 1337][note.deep:some note] message
37−[2016-06-12 12:34:56][foo][Info][example][7331][1337][note.deep:some note][main:Some.Module path/Some/Module.hs:30:1] message
37+[2016-06-12 12:34:56][foo][Info][example][PID 7331][ThreadId 1337][note.deep:some note][main:Some.Module path/Some/Module.hs:30:1] message
38−[2016-06-12 12:34:56][foo][Info][example][7331][1337][note.deep:some note][main:Some.Module путь/Some/Module.hs:3000:9000] message
38+[2016-06-12 12:34:56][foo][Info][example][PID 7331][ThreadId 1337][note.deep:some note][main:Some.Module путь/Some/Module.hs:3000:9000] message
39−[2016-06-12 12:34:56][foo][Info][example][7331][1337][sub:null][text:][num:0][float:0] message
39+[2016-06-12 12:34:56][foo][Info][example][PID 7331][ThreadId 1337][sub:null][text:][num:0][float:0] message
40−[2016-06-12 12:34:56][foo][Info][example][7331][1337][sub.note.deep:some note][text:note][num:10][float:5.5] message
40+[2016-06-12 12:34:56][foo][Info][example][PID 7331][ThreadId 1337][sub.note.deep:some note][text:note][num:10][float:5.5] message
katip/test/Katip/Tests/Scribes/Handle.hs+20−10
@@ -1,4 +1,5 @@
1{-# LANGUAGE OverloadedStrings #-}
1{-# LANGUAGE OverloadedStrings #-}
2+{-# LANGUAGE RankNTypes #-}
2module Katip.Tests.Scribes.Handle
3module Katip.Tests.Scribes.Handle
3 ( tests
4 ( tests
4 ) where
5 ) where
@@ -6,9 +7,9 @@ module Katip.Tests.Scribes.Handle
6-------------------------------------------------------------------------------
7-------------------------------------------------------------------------------
7import Control.Monad
8import Control.Monad
8import Data.Aeson
9import Data.Aeson
9−import qualified Data.ByteString.Lazy as BL
10import qualified Data.ByteString.Char8 as B
10import qualified Data.ByteString.Char8 as B
11+import qualified Data.ByteString.Lazy as BL
11−import Data.Monoid
12+import Data.Monoid as M
12import Data.Text (Text)
13import Data.Text (Text)
13import qualified Data.Text as T
14import qualified Data.Text as T
14import qualified Data.Text.Encoding as T
15import qualified Data.Text.Encoding as T
@@ -25,7 +26,6 @@ import Test.Tasty.HUnit
25import Text.Regex.TDFA
26import Text.Regex.TDFA
26-------------------------------------------------------------------------------
27-------------------------------------------------------------------------------
27import Katip
28import Katip
28−import Katip.Scribes.Handle
29-------------------------------------------------------------------------------
29-------------------------------------------------------------------------------
30
30
31
31
@@ -39,22 +39,26 @@ tests = testGroup "Katip.Scribes.Handle"
39 runKatipT le $ logItem dummyLogItem "test" Nothing InfoS "wont make it in"
39 runKatipT le $ logItem dummyLogItem "test" Nothing InfoS "wont make it in"
40 hClose h
40 hClose h
41 res <- readFile path
41 res <- readFile path
42− let pat = "\\[[[:digit:]]{4}-[[:digit:]]{2}-[[:digit:]]{2} [[:digit:]]{2}:[[:digit:]]{2}:[[:digit:]]{2}\\]\\[katip-test.test\\]\\[Info\\]\\[.+\\]\\[[[:digit:]]+\\]\\[ThreadId [[:digit:]]+\\]\\[note.deep:some note\\] test message" :: String
42+ let pat = "\\[[[:digit:]]{4}-[[:digit:]]{2}-[[:digit:]]{2} [[:digit:]]{2}:[[:digit:]]{2}:[[:digit:]]{2}\\]\\[katip-test.test\\]\\[Info\\]\\[.+\\]\\[[PID [:digit:]]+\\]\\[ThreadId [[:digit:]]+\\]\\[note.deep:some note\\] test message" :: String
43 let matches = res =~ pat
43 let matches = res =~ pat
44− assertBool (show res <> " did not match") matches
44+ assertBool (show res M.<> " did not match") matches
45 , withResource setupFile (const (return ())) $ \setupScribe -> testCase "logs correct data to a file" $ do
45 , withResource setupFile (const (return ())) $ \setupScribe -> testCase "logs correct data to a file" $ do
46 (path, fin, le) <- setupScribe
46 (path, fin, le) <- setupScribe
47 runKatipT le $ logItem dummyLogItem "test" Nothing InfoS "test message"
47 runKatipT le $ logItem dummyLogItem "test" Nothing InfoS "test message"
48 fin
48 fin
49 runKatipT le $ logItem dummyLogItem "test" Nothing InfoS "wont make it in"
49 runKatipT le $ logItem dummyLogItem "test" Nothing InfoS "wont make it in"
50 res <- readFile path
50 res <- readFile path
51− let pat = "\\[[[:digit:]]{4}-[[:digit:]]{2}-[[:digit:]]{2} [[:digit:]]{2}:[[:digit:]]{2}:[[:digit:]]{2}\\]\\[katip-test.test\\]\\[Info\\]\\[.+\\]\\[[[:digit:]]+\\]\\[ThreadId [[:digit:]]+\\]\\[note.deep:some note\\] test message" :: String
51+ let pat = "\\[[[:digit:]]{4}-[[:digit:]]{2}-[[:digit:]]{2} [[:digit:]]{2}:[[:digit:]]{2}:[[:digit:]]{2}\\]\\[katip-test.test\\]\\[Info\\]\\[.+\\]\\[[PID [:digit:]]+\\]\\[ThreadId [[:digit:]]+\\]\\[note.deep:some note\\] test message" :: String
52 let matches = res =~ pat
52 let matches = res =~ pat
53 assertBool (show res <> " did not match") matches
53 assertBool (show res <> " did not match") matches
54 , withResource setupTempFile teardownTempFile $ \setupFn ->
54 , withResource setupTempFile teardownTempFile $ \setupFn ->
55 goldenVsString "Text-golden"
55 goldenVsString "Text-golden"
56 "test/Katip/Tests/Scribes/Handle-text.golden"
56 "test/Katip/Tests/Scribes/Handle-text.golden"
57 (setupFn >>= writeTextLog)
57 (setupFn >>= writeTextLog)
58+ , withResource setupTempFile teardownTempFile $ \setupFn ->
59+ goldenVsString "Json-golden"
60+ "test/Katip/Tests/Scribes/Handle-json.golden"
61+ (setupFn >>= writeJsonLog)
58 ]
62 ]
59
63
60
64
@@ -88,7 +92,7 @@ setup :: IO (FilePath, Handle, IO (), LogEnv)
88setup = do
92setup = do
89 tempDir <- getTemporaryDirectory
93 tempDir <- getTemporaryDirectory
90 (fp, h) <- openTempFile tempDir "katip.log"
94 (fp, h) <- openTempFile tempDir "katip.log"
91− s <- mkHandleScribe (ColorLog False) h DebugS V3
95+ s <- mkHandleScribe (ColorLog False) h (permitItem DebugS) V3
92 le <- initLogEnv "katip-test" "test"
96 le <- initLogEnv "katip-test" "test"
93 le' <- registerScribe "handle" s defaultScribeSettings le
97 le' <- registerScribe "handle" s defaultScribeSettings le
94 return (fp, h, void (closeScribes le'), le')
98 return (fp, h, void (closeScribes le'), le')
@@ -107,7 +111,7 @@ setupFile = do
107 tempDir <- getTemporaryDirectory
111 tempDir <- getTemporaryDirectory
108 (fp, h) <- openTempFile tempDir "katip.log"
112 (fp, h) <- openTempFile tempDir "katip.log"
109 hClose h
113 hClose h
110− s <- mkFileScribe fp DebugS V3
114+ s <- mkFileScribe fp (permitItem DebugS) V3
111 le <- initLogEnv "katip-test" "test"
115 le <- initLogEnv "katip-test" "test"
112 le' <- registerScribe "handle" s defaultScribeSettings le
116 le' <- registerScribe "handle" s defaultScribeSettings le
113 return (fp, void (closeScribes le'), le')
117 return (fp, void (closeScribes le'), le')
@@ -234,14 +238,20 @@ mkUTCTime y mt d h mn s = UTCTime day dt
234
238
235-------------------------------------------------------------------------------
239-------------------------------------------------------------------------------
236writeTextLog :: (FilePath, Handle) -> IO (BL.ByteString)
240writeTextLog :: (FilePath, Handle) -> IO (BL.ByteString)
237−writeTextLog (path, h) = do
241+writeTextLog = writeFormattedLog bracketFormat
242+
243+writeJsonLog :: (FilePath, Handle) -> IO (BL.ByteString)
244+writeJsonLog = writeFormattedLog jsonFormat
245+
246+writeFormattedLog :: (forall a . LogItem a => ItemFormatter a) -> (FilePath, Handle) -> IO (BL.ByteString)
247+writeFormattedLog format (path, h) = do
238 mapM_ (put . formatOne) genItems
248 mapM_ (put . formatOne) genItems
239 mapM_ (put . formatOne) genTypedItems
249 mapM_ (put . formatOne) genTypedItems
240 hClose h
250 hClose h
241 BL.readFile path
251 BL.readFile path
242 where
252 where
243 formatOne :: LogItem a => Item a -> Text
253 formatOne :: LogItem a => Item a -> Text
244− formatOne = LT.toStrict . LT.toLazyText . formatItem False V3
254+ formatOne = LT.toStrict . LT.toLazyText . format False V3
245 put = B.hPutStrLn h . T.encodeUtf8
255 put = B.hPutStrLn h . T.encodeUtf8
246
256
247setupTempFile :: IO (FilePath, Handle)
257setupTempFile :: IO (FilePath, Handle)
stack-lts-2.yamldeleted−32
@@ -1,32 +0,0 @@
1−resolver: lts-2.22
2−packages:
3− - katip
4− - katip-elasticsearch
5−extra-deps:
6− - bloodhound-0.14.0.0
7− - string-conv-0.1.2
8− - aeson-0.11.1.0
9− - http-client-0.5.4
10− - fail-4.9.0.0
11− - tagged-0.8.3
12− - uuid-1.3.12
13− - attoparsec-0.13.1.0
14− - retry-0.7.4.2
15− - tasty-0.11.0.4
16− - time-locale-compat-0.1.1.1
17− - cryptonite-0.20
18− - microlens-0.4.7.0
19− - microlens-th-0.4.1.0
20− - time-1.5.0.1
21− - memory-0.13
22− - rng-utils-0.2.1
23− - lens-aeson-1.0.0.5
24− - atomic-primops-0.8.0.4
25− - safe-exceptions-0.1.5.0
26−flags:
27− katip:
28− lib-Werror: true
29− katip-elasticsearch:
30− lib-Werror: true
31− time-locale-compat:
32− old-locale: false
stack-lts-3.yamldeleted−22
@@ -1,22 +0,0 @@
1−resolver: lts-3.22
2−packages:
3− - katip
4− - katip-elasticsearch
5−extra-deps:
6− - bloodhound-0.14.0.0
7− - string-conv-0.1.2
8− - aeson-0.11.1.0
9− - http-client-0.5.4
10− - fail-4.9.0.0
11− - tagged-0.8.3
12− - uuid-1.3.12
13− - attoparsec-0.13.1.0
14− - retry-0.7.4.2
15− - tasty-0.11.0.4
16− - rng-utils-0.2.1
17− - safe-exceptions-0.1.5.0
18−flags:
19− katip:
20− lib-Werror: true
21− katip-elasticsearch:
22− lib-Werror: true
stack-lts-4.yamldeleted−19
@@ -1,19 +0,0 @@
1−resolver: lts-4.2
2−packages:
3− - katip
4− - katip-elasticsearch
5−extra-deps:
6− - bloodhound-0.14.0.0
7− - string-conv-0.1.2
8− - aeson-0.11.1.0
9− - http-client-0.5.4
10− - fail-4.9.0.0
11− - tagged-0.8.3
12− - uuid-1.3.12
13− - rng-utils-0.2.1
14− - safe-exceptions-0.1.5.0
15−flags:
16− katip:
17− lib-Werror: true
18− katip-elasticsearch:
19− lib-Werror: true
stack-lts-5.yamldeleted−16
@@ -1,16 +0,0 @@
1−resolver: lts-5.18
2−packages:
3− - katip
4− - katip-elasticsearch
5−extra-deps:
6− - bloodhound-0.14.0.0
7− - string-conv-0.1.2
8− - aeson-0.11.1.0
9− - http-client-0.5.4
10− - rng-utils-0.2.1
11− - safe-exceptions-0.1.5.0
12−flags:
13− katip:
14− lib-Werror: true
15− katip-elasticsearch:
16− lib-Werror: true
stack.yaml+11−2
@@ -1,11 +1,20 @@
1−resolver: lts-8.21
1+resolver: lts-16.18
2packages:
2packages:
3 - katip
3 - katip
4 - katip-elasticsearch
4 - katip-elasticsearch
5+ - katip-datadog
6+ - katip-logzio
5extra-deps:
7extra-deps:
8+ # ghc 8.8 support
6− - bloodhound-0.15.0.0
9+ - github: wireapp/bloodhound
10+ commit: b409f52b518f20a4c20651b5893c31b7b85cb9b1
11+allow-newer: true
7flags:
12flags:
8 katip:
13 katip:
9 lib-Werror: true
14 lib-Werror: true
10 katip-elasticsearch:
15 katip-elasticsearch:
11 lib-Werror: true
16 lib-Werror: true
17+ katip-datadog:
18+ lib-Werror: true
19+ katip-logzio:
20+ lib-Werror: true