on master help sign in
44 files changed+2996−504
since splitsnapshots
unifiedsplit
.gitignore+1 .travis.yml+16−10 katip-datadog/LICENSE+30 katip-datadog/README.md+6 katip-datadog/changelog.md+7 katip-datadog/katip-datadog.cabal+78 katip-datadog/src/Katip/Scribes/Datadog/TCP.hs+191 katip-datadog/test/Katip/Tests/Scribes/Datadog/TCP.hs+105 katip-datadog/test/Main.hs+22 katip-elasticsearch/bench/Main.hs+12−15 katip-elasticsearch/changelog.md+69 katip-elasticsearch/docker-compose.yml+10 katip-elasticsearch/examples/example.hs+34 katip-elasticsearch/katip-elasticsearch.cabal+29−25 katip-elasticsearch/src/Katip/Scribes/ElasticSearch.hs+49−1 katip-elasticsearch/src/Katip/Scribes/ElasticSearch/Annotations.hs+2−2 katip-elasticsearch/src/Katip/Scribes/ElasticSearch/Internal.hs+101−44 katip-elasticsearch/test/Main.hs+35−10 katip-logzio/LICENSE+30 katip-logzio/README.md+4 katip-logzio/changelog.md+9 katip-logzio/katip-logzio.cabal+94 katip-logzio/src/Katip/Scribes/LogzIO/HTTPS.hs+639 katip-logzio/test/Katip/Tests/Scribes/LogzIO/HTTPS.hs+466 katip-logzio/test/Main.hs+22 katip/README.md+4−1 katip/bench/Main.hs+5−2 katip/changelog.md+114 katip/examples/example.hs+24−22 katip/examples/example_lens.hs+23−22 katip/katip.cabal+28−26 katip/src/Katip.hs+92−9 katip/src/Katip/Core.hs+248−110 katip/src/Katip/Monadic.hs+120−25 katip/src/Katip/Scribes/Handle.hs+100−30 katip/test/Katip/Tests.hs+48−9 katip/test/Katip/Tests/Scribes/Handle-json.golden+38 katip/test/Katip/Tests/Scribes/Handle-text.golden+38−38 katip/test/Katip/Tests/Scribes/Handle.hs+42−12 stack-lts-2.yaml−32 stack-lts-3.yaml−22 stack-lts-4.yaml−19 stack-lts-5.yaml−16 stack.yaml+11−2
.gitignore+1
@@ -7,3 +7,4 @@ TAGS
77*.hi
88*.dyn_o
99*.dyn_hi
10+stack.yaml.lock
.travis.yml+16−10
@@ -9,21 +9,19 @@ addons:
99 apt:
1010 packages:
1111 - libgmp-dev
12− - default-jre # needed for java 8
1312
1413# The different configurations we want to test. You could also do things like
1514# change flags or use --stack-yaml to point to a different file.
1615matrix:
1716 include:
18− # - env: ARGS="--stack-yaml=stack-lts-2.yaml" ESVERSION=1.5.2 ESURL=https://download.elastic.co/elasticsearch/elasticsearch ESTESTARGS="--es-version 1"
19− # - env: ARGS="--stack-yaml=stack-lts-3.yaml" ESVERSION=1.5.2 ESURL=https://download.elastic.co/elasticsearch/elasticsearch ESTESTARGS="--es-version 1"
20− # - env: ARGS="--stack-yaml=stack-lts-4.yaml" ESVERSION=1.5.2 ESURL=https://download.elastic.co/elasticsearch/elasticsearch ESTESTARGS="--es-version 1"
21− # - env: ARGS="--stack-yaml=stack-lts-5.yaml" ESVERSION=1.5.2 ESURL=https://download.elastic.co/elasticsearch/elasticsearch ESTESTARGS="--es-version 1"
22− # - env: ARGS="--resolver=lts-6" ESVERSION=1.5.2 ESURL=https://download.elastic.co/elasticsearch/elasticsearch ESTESTARGS="--es-version 1"
23− # - env: ARGS="--resolver=lts-7" ESVERSION=1.5.2 ESURL=https://download.elastic.co/elasticsearch/elasticsearch ESTESTARGS="--es-version 1"
24− # - env: ARGS="--resolver=lts-8" ESVERSION=1.5.2 ESURL=https://download.elastic.co/elasticsearch/elasticsearch ESTESTARGS="--es-version 1"
25− # - env: ARGS="--resolver=nightly" ESVERSION=1.5.2 ESURL=https://download.elastic.co/elasticsearch/elasticsearch ESTESTARGS="--es-version 1"
26− - env: ARGS="--resolver=nightly" ESVERSION=5.5.0 ESURL=https://artifacts.elastic.co/downloads/elasticsearch ESTESTARGS="--es-version 5"
17+ - env: ARGS="--resolver=lts-16" ESVERSION=5.5.0 ESURL=https://artifacts.elastic.co/downloads/elasticsearch ESTESTARGS="--es-version 5" ES_JAVA_OPTS="-Xms500m -Xmx750m"
18+ - env: ARGS="--resolver=lts-16" ESVERSION=6.2.2 ESURL=https://artifacts.elastic.co/downloads/elasticsearch ESTESTARGS="--es-version 5" ES_JAVA_OPTS="-Xms500m -Xmx750m"
19+ - env: ARGS="--resolver=nightly" ESVERSION=5.5.0 ESURL=https://artifacts.elastic.co/downloads/elasticsearch ESTESTARGS="--es-version 5" ES_JAVA_OPTS="-Xms500m -Xmx750m"
20+ - env: ARGS="--resolver=nightly" ESVERSION=6.2.2 ESURL=https://artifacts.elastic.co/downloads/elasticsearch ESTESTARGS="--es-version 5" ES_JAVA_OPTS="-Xms500m -Xmx750m"
21+ # nightlies may fail usually due to lagging dependencies
22+ allow_failures:
23+ - env: ARGS="--resolver=nightly" ESVERSION=5.5.0 ESURL=https://artifacts.elastic.co/downloads/elasticsearch ESTESTARGS="--es-version 5" ES_JAVA_OPTS="-Xms500m -Xmx750m"
24+ - env: ARGS="--resolver=nightly" ESVERSION=6.2.2 ESURL=https://artifacts.elastic.co/downloads/elasticsearch ESTESTARGS="--es-version 5" ES_JAVA_OPTS="-Xms500m -Xmx750m"
2725
2826before_install:
2927# Download and unpack the stack executable
@@ -34,7 +32,11 @@ before_install:
3432install:
3533 - wget --no-check-certificate $ESURL/elasticsearch-$ESVERSION.tar.gz
3634 - tar xzf elasticsearch-$ESVERSION.tar.gz
35+ # set up a repo for snapshot testing. Required in ES >= 1.6
36+ - echo "path.repo = [\"/tmp\"]" >> ./elasticsearch-$ESVERSION/elasticsearch.yml
37+ - java -version
3738 - ./elasticsearch-$ESVERSION/bin/elasticsearch &
39+ # Bloodhound seems to take a while to build and can sometimes stall out the build when travis detects no output.
3840
3941
4042# This line does all of the work: installs GHC if necessary, build the library,
@@ -42,14 +44,18 @@ install:
4244# around some quirks in Travis's terminal implementation.
4345script:
4446 - stack $ARGS setup
47+ # Prevent Inactivity timeouts. See https://github.com/travis-ci/travis-ci/issues/7020#issuecomment-288961580
48+ - travis_wait 45 sleep 1800 &
4549 - stack $ARGS test katip:test --no-terminal --haddock --no-haddock-deps
4650 - stack $ARGS test katip-elasticsearch:test --no-terminal --test-arguments="$ESTESTARGS"
4751 - stack $ARGS bench
4852 - stack $ARGS build
4953 - for f in katip/examples/*.hs; do stack $ARGS ghc -- -Wall -Werror $f -o /dev/null; done
54+ - for f in katip-elasticsearch/examples/*.hs; do stack $ARGS ghc -- -Wall -Werror $f -o /dev/null; done
5055 - stack $ARGS sdist
5156
5257# Caching so the next build will be fast too.
5358cache:
5459 directories:
5560 - $HOME/.stack
61+ - .stack-work
katip-datadog/LICENSEnew file+30
@@ -0,0 +1,30 @@
1+Copyright (c) 2018, Soostone Inc
2+
3+All rights reserved.
4+
5+Redistribution and use in source and binary forms, with or without
6+modification, are permitted provided that the following conditions are met:
7+
8+ * Redistributions of source code must retain the above copyright
9+ notice, this list of conditions and the following disclaimer.
10+
11+ * Redistributions in binary form must reproduce the above
12+ copyright notice, this list of conditions and the following
13+ disclaimer in the documentation and/or other materials provided
14+ with the distribution.
15+
16+ * Neither the name of Ozgun Ataman nor the names of other
17+ contributors may be used to endorse or promote products derived
18+ from this software without specific prior written permission.
19+
20+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
21+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
22+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
23+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
24+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
25+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
26+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
27+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
28+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
29+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
30+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
katip-datadog/README.mdnew file+6
@@ -0,0 +1,6 @@
1+# Katip Datadog [![Build Status](https://travis-ci.org/Soostone/katip.svg?branch=master)](https://travis-ci.org/Soostone/katip)
2+
3+katip-datadog is a scribe for the Katip logging framework that sends
4+structured logs to the Datadog logging service. It supports both
5+directly sending to the ingestion API and through a locally installed
6+datadog agent.
katip-datadog/changelog.mdnew file+7
@@ -0,0 +1,7 @@
1+0.2.0.0
2+==========
3+* Remove import of deprecated Network module
4+
5+0.1.0.0
6+==========
7+* Initial release
katip-datadog/katip-datadog.cabalnew file+78
@@ -0,0 +1,78 @@
1+name: katip-datadog
2+version: 0.2.0.0
3+synopsis: Datadog scribe for the Katip logging framework
4+description: See README.md for more details.
5+license: BSD3
6+license-file: LICENSE
7+author: Michael Xavier
8+maintainer: michael.xavier@soostone.com
9+copyright: Soostone Inc, 2018
10+category: Data, Text, Logging
11+homepage: https://github.com/Soostone/katip
12+bug-reports: https://github.com/Soostone/katip/issues
13+build-type: Simple
14+cabal-version: >=1.10
15+extra-source-files:
16+ README.md
17+ changelog.md
18+
19+--TODO: fill this in
20+tested-with: GHC == 8.4.3
21+
22+source-repository head
23+ type: git
24+ location: https://github.com/Soostone/katip.git
25+
26+flag lib-Werror
27+ default: False
28+ manual: True
29+
30+
31+library
32+ exposed-modules:
33+ Katip.Scribes.Datadog.TCP
34+ build-depends:
35+ base >=4.4.0.0 && <5
36+ , aeson >= 1.0.0.0
37+ , bytestring
38+ , katip >= 0.8.0.0
39+ , text
40+ , connection
41+ , resource-pool
42+ , time
43+ , binary
44+ , network
45+ , safe-exceptions
46+ , retry >= 0.7
47+ hs-source-dirs: src
48+ default-language: Haskell2010
49+ hs-source-dirs: src
50+ ghc-options: -Wall
51+ if flag(lib-Werror)
52+ ghc-options: -Werror
53+
54+
55+test-suite test
56+ type: exitcode-stdio-1.0
57+ main-is: Main.hs
58+ hs-source-dirs: test
59+ default-language: Haskell2010
60+ other-modules:
61+ Katip.Tests.Scribes.Datadog.TCP
62+ build-depends: base
63+ , katip
64+ , katip-datadog
65+ , conduit
66+ , conduit-extra
67+ , tasty
68+ , tasty-hunit
69+ , aeson
70+ , unordered-containers
71+ , containers
72+ , safe-exceptions
73+ , async
74+ , attoparsec
75+ , text
76+ ghc-options: -Wall
77+ if flag(lib-Werror)
78+ ghc-options: -Werror
katip-datadog/src/Katip/Scribes/Datadog/TCP.hsnew file+191
@@ -0,0 +1,191 @@
1+{-# LANGUAGE BangPatterns #-}
2+{-# LANGUAGE OverloadedStrings #-}
3+{-# LANGUAGE RankNTypes #-}
4+-- | Creates a scribe as a Custom Forwarder for Datadog that sends log
5+-- messages over TCP via their TLS endpoint.
6+module Katip.Scribes.Datadog.TCP
7+ ( -- * Types
8+ APIKey(..)
9+ , DatadogScribeSettings(..)
10+ , DatadogAuth(..)
11+ , directAPIConnectionParams
12+ , localAgentConnectionParams
13+ -- * Scribe construction
14+ , mkDatadogScribeSettings
15+ , mkDatadogScribe
16+ ) where
17+
18+
19+
20+-------------------------------------------------------------------------------
21+import qualified Control.Concurrent as Conc
22+import qualified Control.Exception.Safe as EX
23+import Control.Monad (void)
24+import qualified Control.Retry as Retry
25+import qualified Data.Aeson as A
26+import qualified Data.Binary.Builder as BB
27+import qualified Data.ByteString.Lazy as BSL
28+import Data.Monoid as Monoid
29+import qualified Data.Pool as Pool
30+import Data.Text (Text)
31+import qualified Data.Text.Encoding as T
32+import qualified Data.Text.Lazy.Builder as B
33+import Data.Time (NominalDiffTime)
34+import Katip
35+import Katip.Core (LocJs (..))
36+import qualified Network.Connection as C
37+import qualified Network.Socket as Net
38+import qualified System.Posix.Types as POSIX
39+-------------------------------------------------------------------------------
40+
41+
42+-- | When writing directly to the main intake API, you must specify an
43+-- API key
44+newtype APIKey = APIKey
45+ { apiKey :: Text
46+ } deriving (Show, Eq)
47+
48+
49+data DatadogAuth =
50+ NoAuthLocal
51+ -- ^ Writing to the local agent, no auth required
52+ | DirectAuth APIKey
53+ -- ^ Writing directly to the main intake API over TLS . API Key is
54+ -- required. Using a local agent is recommended.
55+
56+
57+data DatadogScribeSettings = DatadogScribeSettings
58+ { datadogScribeSettings_connectionParams :: C.ConnectionParams
59+ -- ^ Specify where the logs should go. If writing directly to the
60+ -- main intake API, you can use 'directAPIConnectionParams'. If
61+ -- writing to a local agent, we recommend
62+ -- 'localAgentConnectionParams'
63+ , datadogScribeSettings_auth :: DatadogAuth
64+ , datadogScribeSettings_poolStripes :: Int
65+ -- ^ How many stripes should be used in the connection pool. 1 is a reasonable default
66+ , datadogScribeSettings_connsPerStripe :: Int
67+ -- ^ How many TCP connections per stripe should be maintained in the pool.
68+ , datadogScribeSettings_connectionKeepalive :: NominalDiffTime
69+ -- ^ How long should a TCP connection be kept open. 30 (seconds) is a reasonable default. Anything around 60 will tend to time out a lot.
70+ , datadogScribeSettings_retry :: Retry.RetryPolicyM IO
71+ -- ^ How should exceptions during write be retried?
72+ }
73+
74+
75+-- | Configured for secure TLS communication to the main intake API at
76+-- intake.logs.datadoghq.com:10516. Must be combined with DirectAuth
77+-- authentication.
78+directAPIConnectionParams :: C.ConnectionParams
79+directAPIConnectionParams = C.ConnectionParams
80+ { C.connectionHostname = "intake.logs.datadoghq.com"
81+ , C.connectionPort = 10516
82+ , C.connectionUseSecure = Just $ C.TLSSettingsSimple
83+ { C.settingDisableCertificateValidation = False
84+ , C.settingDisableSession = False
85+ , C.settingUseServerName = False
86+ }
87+ , C.connectionUseSocks = Nothing
88+ }
89+
90+
91+-- | Configure to talk to a local Datadog agent at 127.0.0.1. TLS is not
92+-- utilized. This is the preferred type of connection because
93+-- authentication, buffering, and other more advanced features are
94+-- handled by the agent configuration.
95+localAgentConnectionParams :: Net.PortNumber -> C.ConnectionParams
96+localAgentConnectionParams port = C.ConnectionParams
97+ { C.connectionHostname = "127.0.0.1"
98+ , C.connectionPort = port
99+ , C.connectionUseSecure = Nothing
100+ , C.connectionUseSocks = Nothing
101+ }
102+
103+
104+-- | Reasonable defaults. Sets a 30s connection timeout, 1 stripe of
105+-- connections with a number of connections per stripe equal to
106+-- 'getNumCapabilities'. Retry policy will do an exponential backoff
107+-- with a 25ms base delay for up to 5 retries for a total cumulative delay of 775ms
108+mkDatadogScribeSettings :: C.ConnectionParams -> DatadogAuth -> IO DatadogScribeSettings
109+mkDatadogScribeSettings connectionParams auth = do
110+ capabilities <- Conc.getNumCapabilities
111+ pure $ DatadogScribeSettings
112+ { datadogScribeSettings_connectionParams = connectionParams
113+ , datadogScribeSettings_auth = auth
114+ , datadogScribeSettings_poolStripes = 1
115+ , datadogScribeSettings_connsPerStripe = capabilities
116+ , datadogScribeSettings_connectionKeepalive = 30
117+ , datadogScribeSettings_retry = Retry.exponentialBackoff 25000 <> Retry.limitRetries 5
118+ }
119+
120+
121+-------------------------------------------------------------------------------
122+mkDatadogScribe
123+ :: DatadogScribeSettings
124+ -> PermitFunc
125+ -- ^ Function on whether to permit items, e.g. @permitItem InfoS@
126+ -> Verbosity
127+ -- ^ Verbosity level to observe
128+ -> IO Scribe
129+mkDatadogScribe settings permit verb = do
130+ connectionContext <- C.initConnectionContext
131+ pool <- Pool.createPool
132+ (C.connectTo connectionContext (datadogScribeSettings_connectionParams settings))
133+ C.connectionClose
134+ (datadogScribeSettings_poolStripes settings)
135+ (datadogScribeSettings_connectionKeepalive settings)
136+ (datadogScribeSettings_connsPerStripe settings)
137+ pure $ Scribe
138+ { liPush = pushPool (datadogScribeSettings_retry settings) pool keyBuilder verb
139+ , scribeFinalizer = Pool.destroyAllResources pool
140+ , scribePermitItem = permit
141+ }
142+ where
143+ !keyBuilder = case datadogScribeSettings_auth settings of
144+ NoAuthLocal -> Nothing
145+ DirectAuth (APIKey k) -> Just (BB.fromByteString (T.encodeUtf8 k))
146+
147+
148+pushPool
149+ :: LogItem a
150+ => Retry.RetryPolicyM IO
151+ -> Pool.Pool C.Connection
152+ -> Maybe BB.Builder
153+ -- ^ Rendered API key
154+ -> Verbosity
155+ -> Item a
156+ -> IO ()
157+pushPool retryPolicy pool token verb item =
158+ void $ Retry.retrying retryPolicy (\_status shouldRetry -> pure shouldRetry) $ \_status -> do
159+ res <- EX.tryAny $ Pool.withResource pool $ \conn -> do
160+ C.connectionPut conn rendered
161+ pure $ case res of
162+ Left _ -> True
163+ Right _ -> False
164+ where
165+ payloadBuilder = A.fromEncoding (encodeDatadog verb item) Monoid.<> "\n"
166+ messageBuilder = case token of
167+ --TODO: lots of conversion going on here. could use a bench
168+ Nothing -> payloadBuilder
169+ Just directKey -> directKey <> " " <> payloadBuilder
170+ rendered = BSL.toStrict (BB.toLazyByteString messageBuilder)
171+
172+
173+encodeDatadog :: LogItem a => Verbosity -> Item a -> A.Encoding
174+encodeDatadog verb i = A.pairs $
175+ -- datadog seems to use "service" to denote appname
176+ "service" A..= _itemApp i <>
177+ -- datadog uses severity (or some other alternatives for sev)
178+ "severity" A..= _itemSeverity i <>
179+ "env" A..= _itemEnv i <>
180+ "thread" A..= getThreadIdText (_itemThread i) <>
181+ "host" A..= _itemHost i <>
182+ "pid" A..= pidInt <>
183+ "data" A..= payloadObject verb (_itemPayload i) <>
184+ -- datadog uses "message" instead of msg
185+ "message" A..= (B.toLazyText (unLogStr (_itemMessage i))) <>
186+ -- datadog uses "date" instead of "at"
187+ "date" A..= _itemTime i <>
188+ "ns" A..= _itemNamespace i <>
189+ "loc" A..= fmap LocJs (_itemLoc i)
190+ where
191+ POSIX.CPid pidInt = _itemProcess i
katip-datadog/test/Katip/Tests/Scribes/Datadog/TCP.hsnew file+105
@@ -0,0 +1,105 @@
1+{-# LANGUAGE OverloadedStrings #-}
2+module Katip.Tests.Scribes.Datadog.TCP
3+ ( tests
4+ ) where
5+
6+
7+-------------------------------------------------------------------------------
8+import Control.Concurrent.Async
9+import qualified Control.Exception.Safe as EX
10+import qualified Data.Aeson as A
11+import qualified Data.Char as Char
12+import qualified Data.Conduit as C
13+import qualified Data.Conduit.Binary as CB
14+import qualified Data.Conduit.List as CL
15+import qualified Data.Conduit.Network as CN
16+import Data.Foldable (toList)
17+import qualified Data.HashMap.Strict as HM
18+import Data.IORef
19+import Data.Sequence ((|>))
20+import Data.Text (Text)
21+import qualified Data.Text as T
22+import Test.Tasty
23+import Test.Tasty.HUnit
24+-------------------------------------------------------------------------------
25+import Katip
26+import Katip.Scribes.Datadog.TCP
27+-------------------------------------------------------------------------------
28+
29+
30+
31+tests :: TestTree
32+tests = testGroup "Katip.Scribes.Datadog.TCP"
33+ [ testCase "logs well-formed messages in order" $ do
34+ logs <- collectLogs $ do
35+ logItem (sl "foo" ("bar" :: Text)) "mynamespace" noLoc InfoS "a message"
36+ length logs @?= 1
37+ let (CapturedLog ns dta sev msg thread) = head logs
38+ ns @?= Namespace ["katip-datadog-tests", "mynamespace"]
39+ dta @?= A.Object (HM.singleton "foo" (A.String "bar"))
40+ sev @?= InfoS
41+ msg @?= "a message"
42+ if T.all Char.isNumber thread && T.length thread > 0
43+ then pure ()
44+ else assertFailure ("Expected thread id to just be a number but it was " <> T.unpack thread)
45+ ]
46+ where
47+ noLoc = Nothing
48+
49+
50+-------------------------------------------------------------------------------
51+-- | A log type parsed out of the fake datadog agent's buffer
52+data CapturedLog = CapturedLog
53+ { capturedLog_namespace :: Namespace
54+ , capturedLog_data :: A.Value
55+ , capturedLog_sev :: Severity
56+ , capturedLog_message :: Text
57+ , capturedLog_thread :: Text
58+ } deriving (Show, Eq)
59+
60+
61+instance A.FromJSON CapturedLog where
62+ parseJSON = A.withObject "CapturedLog" $ \o -> do
63+ message <- o A..: "message"
64+ sev <- o A..: "severity"
65+ dta <- o A..: "data"
66+ namespace <- o A..: "ns"
67+ thread <- o A..: "thread"
68+ pure $ CapturedLog
69+ { capturedLog_namespace = namespace
70+ , capturedLog_data = dta
71+ , capturedLog_sev = sev
72+ , capturedLog_message = message
73+ , capturedLog_thread = thread
74+ }
75+
76+
77+-------------------------------------------------------------------------------
78+collectLogs :: KatipT IO () -> IO [CapturedLog]
79+collectLogs f = do
80+ capturedRef <- newIORef mempty
81+ withAsync (startServer capturedRef) $ \_worker -> do
82+ EX.bracket mkLE closeLE $ \le -> do
83+ runKatipT le f
84+ toList <$> readIORef capturedRef
85+ where
86+ closeLE = closeScribes
87+ port = 1337
88+ mkLE = do
89+ datadogScribeSettings <- mkDatadogScribeSettings (localAgentConnectionParams (fromInteger port)) NoAuthLocal
90+ scribe <- mkDatadogScribe datadogScribeSettings (permitItem DebugS) V3
91+ registerScribe "datadog" scribe defaultScribeSettings =<< initLogEnv "katip-datadog-tests" "test"
92+ startServer capturedRef = CN.runTCPServer (CN.serverSettings (fromInteger port) "*") $ \appData -> C.runConduit $
93+ CN.appSource appData C..|
94+ CB.lines C..|
95+ CL.map A.eitherDecodeStrict C..|
96+ CL.mapM_ (\parseResult -> case parseResult of
97+ Left e -> EX.throwM (ParseError e)
98+ (Right capturedLog) -> modifyIORef' capturedRef (\captured -> captured |> capturedLog))
99+
100+
101+-------------------------------------------------------------------------------
102+newtype ParseError = ParseError String
103+ deriving (Show)
104+
105+instance EX.Exception ParseError
katip-datadog/test/Main.hsnew file+22
@@ -0,0 +1,22 @@
1+module Main
2+ ( main
3+ ) where
4+
5+
6+-------------------------------------------------------------------------------
7+import Test.Tasty
8+-------------------------------------------------------------------------------
9+import qualified Katip.Tests.Scribes.Datadog.TCP
10+-------------------------------------------------------------------------------
11+
12+
13+
14+main :: IO ()
15+main = defaultMain tests
16+
17+
18+-------------------------------------------------------------------------------
19+tests :: TestTree
20+tests = testGroup "katip-datadog"
21+ [ Katip.Tests.Scribes.Datadog.TCP.tests
22+ ]
katip-elasticsearch/bench/Main.hs+12−15
@@ -13,30 +13,29 @@ import Criterion.Main
1313import Data.Aeson
1414import qualified Data.HashMap.Strict as HM
1515import Data.Proxy (Proxy (..))
16−import Data.RNG
16+import System.Random
1717import qualified Data.Text as T
18−import Database.V1.Bloodhound.Types
18+import Database.Bloodhound.Types
1919import Numeric
2020-------------------------------------------------------------------------------
2121import Katip.Scribes.ElasticSearch
22−import Katip.Scribes.ElasticSearch.Internal (ESV1)
22+import Katip.Scribes.ElasticSearch.Internal (ESV5)
2323-------------------------------------------------------------------------------
2424
2525main :: IO ()
2626main = do
27− rng <- mkRNG
2827 defaultMain
2928 [
30− mkDocIdBenchmark rng
29+ mkDocIdBenchmark
3130 , deannotateValueBenchmark
3231 ]
3332
3433-------------------------------------------------------------------------------
35−mkDocIdBenchmark :: RNG -> Benchmark
36−mkDocIdBenchmark rng = bgroup "mkDocId"
34+mkDocIdBenchmark :: Benchmark
35+mkDocIdBenchmark = bgroup "mkDocId"
3736 [
38− bench "mkDocId (randomIO)" $ nfIO (mkDocId (Proxy :: Proxy ESV1))
39− , bench "mkDocId' (shared )" $ nfIO $ mkDocId' rng
37+ bench "mkDocId (randomIO)" $ nfIO (mkDocId (Proxy :: Proxy ESV5))
38+ , bench "mkDocId' (shared )" $ nfIO mkDocId'
4039 ]
4140
4241
@@ -58,13 +57,11 @@ annotatedValue = Object $ HM.fromList [ ("a::string", String "whatever")
5857 ]
5958
6059-------------------------------------------------------------------------------
61−mkDocId' :: RNG -> IO DocId
62−mkDocId' rng = do
63− is <- withRNG rng $ \gen -> replicateM len $ mk gen
64− return . DocId . T.pack . concatMap (`showHex` "") $ is
60+mkDocId' :: IO DocId
61+mkDocId' = do
62+ is <- replicateM len (randomRIO (0, 15))
63+ return (DocId (T.pack (concatMap (`showHex` "") (is :: [Int]))))
6564 where
6665 len = 32
67− mk :: GenIO -> IO Int
68− mk = uniformR (0,15)
6966
7067deriving instance NFData DocId
katip-elasticsearch/changelog.md+69
@@ -1,3 +1,72 @@
1+0.7.0.0
2+=======
3+* Make compatible with unreleased bloodhound 0.17
4+* Deprecate use of ES version 1 which has been removed from bloodhound
5+ 0.17. All users should move over to ESV5 which should work for
6+ versions 5 and up.
7+* Default configuration to ESV5
8+* Add deprecation warnings for ESV1. Support for ESV1 will be removed
9+ in the future.
10+
11+0.6.0.0
12+=======
13+* Update to katip-0.8.0.0 `PermitFunc` convention. To update, for
14+ example, if you were passing `InfoS` to `mkEsScribe` to filter
15+ severities, instead you should pass `(permitItem InfoS)`.
16+* Update examples to add necessary type signature for the default
17+ scribe config.
18+* Bugfix: the default error retry policy was much more rapid than
19+ documented.
20+* Internally switch to safe-exceptions from encolosed-exceptions.
21+
22+0.5.1.1
23+=======
24+* Loosen katip dep
25+
26+0.5.1.0
27+=======
28+* Loosen deps
29+
30+0.5.0.0
31+=======
32+* Update template for ESv1 to not analyze certain fields like host and
33+ namespace. These are not fields that benefit from tokenization. If
34+ you're on ESv1, after running `mkEsScribe`, the logs template will
35+ be updated to stop analyzing those fields. That means the next
36+ index that is cut (e.g. tomorrow's logs if you're using daily index
37+ sharding) will no longer analyze some fields.
38+
39+0.4.2.0
40+=======
41+* Allow newer bloodhound, aeson, async.
42+
43+0.4.1.0
44+=======
45+* Fix bug where index was created in `mkEsScribe` when it would not be used due to index sharding.
46+* Update some index settings if index already exists and sharding is not used.
47+* For ES V5 and higher, stop using the deprecated (and in 6.x, removed) `string` type for index templates, instead using `text` and `keyword` as appropriate. This makes `katip-elasticsearch` compatible with ES 6.x.
48+
49+0.4.0.4
50+=======
51+* Allow http-types 0.12
52+
53+0.4.0.3
54+=======
55+* Fix benchmark and test builds for stackage nightly
56+
57+0.4.0.2
58+=======
59+* Add repository/homepage info to cabal file
60+
61+0.4.0.1
62+=======
63+* Bump dependencies to allow GHC 8.2.1
64+
65+0.4.0.0
66+=======
67+* Update to bloodhound >= 0.13.0.0. This version adds support for both ElasticSearch versions 1 and 5. Previously, we implicitly supported one and maybe would work on 5. The types in `EsScribeCfg` had to change to be able to specify which version was being targeted.
68+* Improved documentation.
69+
1700.3.1.0
271=======
372* Widen dependency on katip
katip-elasticsearch/docker-compose.ymlnew file+10
@@ -0,0 +1,10 @@
1+version: "3.7"
2+services:
3+ # https://hub.docker.com/_/elasticsearch
4+ elasticsearch:
5+ image: elasticsearch:5-alpine
6+ ports:
7+ - target: 9200
8+ published: 9200
9+ protocol: tcp
10+ mode: host
katip-elasticsearch/examples/example.hsnew file+34
@@ -0,0 +1,34 @@
1+{-# LANGUAGE OverloadedStrings #-}
2+module Main
3+ ( main
4+ ) where
5+
6+
7+-------------------------------------------------------------------------------
8+import Control.Exception
9+import Database.Bloodhound
10+import Network.HTTP.Client
11+-------------------------------------------------------------------------------
12+import Katip
13+import Katip.Scribes.ElasticSearch
14+-------------------------------------------------------------------------------
15+
16+
17+main :: IO ()
18+main = do
19+ mgr <- newManager defaultManagerSettings
20+ let bhe = mkBHEnv (Server "http://localhost:9200") mgr
21+ -- You may need to specify the version in a type signature when customizing your index settings
22+ let _customizedExample = defaultEsScribeCfgV5 { essIndexSettings = IndexSettings (ShardCount 1) (ReplicaCount 0)} :: EsScribeCfg ESV5
23+ esScribe <- mkEsScribe
24+ -- Reasonable for production
25+ defaultEsScribeCfgV5
26+ -- Reasonable for single-node in development
27+ bhe
28+ (IndexName "all-indices-prefixed-with")
29+ (MappingName "application-logs")
30+ (permitItem DebugS)
31+ V3
32+ let mkLogEnv = registerScribe "es" esScribe defaultScribeSettings =<< initLogEnv "MyApp" "production"
33+ bracket mkLogEnv closeScribes $ \le -> runKatipT le $ do
34+ logMsg "ns" InfoS "This goes to elasticsearch"
katip-elasticsearch/katip-elasticsearch.cabal+29−25
@@ -1,21 +1,28 @@
11name: katip-elasticsearch
22synopsis: ElasticSearch scribe for the Katip logging framework.
33description: See README.md for more details.
4−version: 0.3.1.0
4+version: 0.7.0.0
55license: BSD3
66license-file: LICENSE
77author: Ozgun Ataman, Michael Xavier
88maintainer: michael.xavier@soostone.com
9−copyright: Soostone Inc, 2015-2016
10−category: Data
9+copyright: Soostone Inc, 2015-2017
10+category: Data, Text, Logging
11+homepage: https://github.com/Soostone/katip
12+bug-reports: https://github.com/Soostone/katip/issues
1113build-type: Simple
1214cabal-version: >=1.10
1315extra-source-files:
1416 README.md
1517 changelog.md
1618 bench/Main.hs
19+ examples/example.hs
1720 test/Main.hs
18−tested-with: GHC == 7.8.4, GHC== 7.10.3
21+tested-with: GHC == 7.8.4, GHC== 7.10.3, GHC == 8.0.1, GHC == 8.2.2, GHC == 8.4.3
22+
23+source-repository head
24+ type: git
25+ location: https://github.com/Soostone/katip.git
1926
2027flag lib-Werror
2128 default: False
@@ -28,28 +35,28 @@ library
2835 Katip.Scribes.ElasticSearch.Internal
2936 build-depends:
3037 base >=4.6 && <5
31− , katip >= 0.2.0.0 && < 0.5
32− , bloodhound >= 0.13.0.0 && < 0.15
33− , uuid >= 1.3.12 && < 1.4
34− , aeson >=0.6 && <1.2
35− , stm >= 2.4.3 && < 2.5
36− , stm-chans >= 3.0.0.2 && < 3.1
37− , async >= 2.0.1.0 && < 2.2
38− , enclosed-exceptions >= 1.0.0 && < 1.1
39− , http-client >= 0.3 && < 0.6
40− , text >= 0.11 && < 1.3
41− , retry >= 0.7 && < 0.8
38+ , katip >= 0.8.0.0
39+ , bloodhound >= 0.13.0.0
40+ , uuid >= 1.3.12
41+ , aeson >=0.6
42+ , stm >= 2.4.3
43+ , stm-chans >= 3.0.0.2
44+ , async >= 2.0.1.0
45+ , safe-exceptions >= 0.1.0.0
46+ , http-client >= 0.3
47+ , text >= 0.11
48+ , retry >= 0.7
4249 , exceptions
43− , scientific >= 0.3.0.0 && < 0.4
44− , unordered-containers >= 0.1.0.0 && < 0.3
45− , transformers >= 0.2 && < 0.6
46− , http-types >= 0.8 && < 0.10
47− , time >= 1 && < 1.7
50+ , scientific >= 0.3.0.0
51+ , unordered-containers >= 0.1.0.0
52+ , transformers >= 0.2
53+ , http-types >= 0.8
54+ , time >= 1
4855 , bytestring
56+ , semigroups
4957 hs-source-dirs: src
5058 default-language: Haskell2010
5159 hs-source-dirs: src
52− default-language: Haskell2010
5360 if flag(lib-Werror)
5461 ghc-options: -Wall -Werror
5562
@@ -65,7 +72,7 @@ benchmark bench
6572 , bloodhound
6673 , deepseq
6774 , criterion >= 1.1.0.0
68− , rng-utils
75+ , random
6976 , uuid
7077 , text
7178 , unordered-containers
@@ -101,7 +108,4 @@ test-suite test
101108 , bytestring
102109 , tagged
103110
104− if flag(lib-Werror)
105− ghc-options: -Wall -Werror
106−
107111 ghc-options: -threaded -rtsopts
katip-elasticsearch/src/Katip/Scribes/ElasticSearch.hs+49−1
@@ -1,10 +1,41 @@
1+{-# LANGUAGE CPP #-}
12-- | Includes a scribe that can be used to log structured, JSON log
23-- messages to ElasticSearch. These logs can be explored easily using
34-- <https://www.elastic.co/products/kibana kibana> or your tool of
45-- choice. Supports ElasticSearch servers with version 1.x or 5.x by
56-- way of different configs.
67--
7−-- == __Important Note on Index Settings__
8+-- Example of configuring for ES5:
9+--
10+-- @
11+--
12+-- import Control.Exception
13+-- import Database.V5.Bloodhound
14+-- import Network.HTTP.Client
15+-- import Katip
16+-- import Katip.Scribes.ElasticSearch
17+--
18+-- main :: IO ()
19+-- main = do
20+-- mgr <- newManager defaultManagerSettings
21+-- let bhe = mkBHEnv (Server "http://localhost:9200") mgr
22+-- esScribe <- mkEsScribe
23+-- -- Reasonable for production
24+-- defaultEsScribeCfgV5
25+-- -- Reasonable for single-node in development
26+-- -- defaultEsScribeCfgV5 { essIndexSettings = IndexSettings (ShardCound 1) (ReplicaCount 0)} :: EsScribeCfg ESV5
27+-- bhe
28+-- (IndexName "all-indices-prefixed-with")
29+-- (MappingName "application-logs")
30+-- DebugS
31+-- V3
32+-- let mkLogEnv = registerScribe "es" esScribe defaultScribeSettings =<< initLogEnv \"MyApp\" "production"
33+-- bracket mkLogEnv closeScribes $ \\le -> runKatipT le $ do
34+-- logMsg "ns" InfoS "This goes to elasticsearch"
35+--
36+-- @
37+--
38+-- __Important Note on Index Settings__
839--
940-- 'defaultEsScribeCfg' inherits a set of default index settings from
1041-- the @bloodhound@ package. These settings at this time of writing
@@ -37,8 +68,17 @@ module Katip.Scribes.ElasticSearch
3768 , essIndexSettings
3869 , essIndexSharding
3970 , defaultEsScribeCfg
71+#if !MIN_VERSION_bloodhound(0,17,0)
4072 , defaultEsScribeCfgV1
73+#endif
4174 , defaultEsScribeCfgV5
75+ -- ** Version-Proxied APIS
76+ -- $versionproxies
77+ , defaultEsScribeCfg'
78+#if !MIN_VERSION_bloodhound(0,17,0)
79+ , ESV1
80+#endif
81+ , ESV5
4282 -- * Utilities
4383 , mkDocId
4484 , module Katip.Scribes.ElasticSearch.Annotations
@@ -49,3 +89,11 @@ module Katip.Scribes.ElasticSearch
4989import Katip.Scribes.ElasticSearch.Annotations
5090import Katip.Scribes.ElasticSearch.Internal
5191-------------------------------------------------------------------------------
92+
93+
94+{- $versionproxies
95+
96+ You may need these these functions and types if type inference
97+ fails. For instance, you may need to hint to the compiler that a
98+ config is @:: EsScribeCfg ESV5@, for instance.
99+-}
katip-elasticsearch/src/Katip/Scribes/ElasticSearch/Annotations.hs+2−2
@@ -12,7 +12,7 @@ import Data.Aeson
1212import qualified Data.Foldable as FT
1313import qualified Data.HashMap.Strict as HM
1414import Data.Maybe
15−import Data.Monoid
15+import Data.Monoid as M
1616import Data.Scientific (isFloating)
1717import Data.Text (Text)
1818import qualified Data.Text as T
@@ -65,7 +65,7 @@ annotateKeys = HM.fromList . map go . HM.toList
6565 where
6666 go (k, Object o) = (k, Object $ annotateKeys o)
6767 go (k, Array a) = (k, Array (annotateValue <$> a))
68− go (k, s@(String _)) = (k <> stringAnn, s)
68+ go (k, s@(String _)) = (k M.<> stringAnn, s)
6969 go (k, n@(Number sci)) = if isFloating sci
7070 then (k <> doubleAnn, n)
7171 else (k <> longAnn, n)
katip-elasticsearch/src/Katip/Scribes/ElasticSearch/Internal.hs+101−44
@@ -16,8 +16,7 @@ import Control.Applicative as A
1616import Control.Concurrent
1717import Control.Concurrent.Async
1818import Control.Concurrent.STM.TBMQueue
19−import Control.Exception.Base
20−import Control.Exception.Enclosed
19+import qualified Control.Exception.Safe as EX
2120import Control.Monad
2221import Control.Monad.Catch
2322import Control.Monad.IO.Class
@@ -28,7 +27,8 @@ import Control.Retry (RetryPolicy,
2827 recovering)
2928import Data.Aeson
3029import Data.ByteString.Lazy (ByteString)
31−import Data.Monoid ((<>))
30+import Data.List.NonEmpty (NonEmpty (..))
31+import Data.Monoid as Monoid
3232import Data.Text (Text)
3333import qualified Data.Text as T
3434import qualified Data.Text.Encoding as T
@@ -37,8 +37,12 @@ import Data.Time.Calendar.WeekDate
3737import Data.Typeable as Typeable
3838import Data.UUID
3939import qualified Data.UUID.V4 as UUID4
40+#if MIN_VERSION_bloodhound(0,17,0)
41+import qualified Database.Bloodhound as V5
42+#else
4043import qualified Database.V1.Bloodhound as V1
4144import qualified Database.V5.Bloodhound as V5
45+#endif
4246import Network.HTTP.Client
4347import Network.HTTP.Types.Status
4448import Text.Printf (printf)
@@ -84,7 +88,7 @@ data EsScribeCfg v = EsScribeCfg {
8488--
8589-- * defaultManagerSettings
8690--
87−-- * exponential backoff with 25ms base delay up to 5 retries
91+-- * exponential backoff with 25ms base delay up to 5 retries, for a total cumulative delay of 775ms
8892--
8993-- * Queue size of 1000
9094--
@@ -95,7 +99,7 @@ data EsScribeCfg v = EsScribeCfg {
9599-- * DailyIndexSharding
96100defaultEsScribeCfg' :: ESVersion v => proxy v -> EsScribeCfg v
97101defaultEsScribeCfg' prx = EsScribeCfg {
98− essRetryPolicy = exponentialBackoff 25 <> limitRetries 5
102+ essRetryPolicy = exponentialBackoff 25000 Monoid.<> limitRetries 5
99103 , essQueueSize = EsQueueSize 1000
100104 , essPoolSize = EsPoolSize 2
101105 , essAnnotateTypes = False
@@ -104,19 +108,22 @@ defaultEsScribeCfg' prx = EsScribeCfg {
104108 }
105109
106110
111+
112+
107113-------------------------------------------------------------------------------
108−-- | Alias of 'defaultEsScribeCfgV1' to minimize API
114+-- | Alias of 'defaultEsScribeCfgV5' to minimize API
109115-- breakage. Previous versions of katip-elasticsearch only supported
110−-- ES version 1.
111−defaultEsScribeCfg :: EsScribeCfg ESV1
112−defaultEsScribeCfg = defaultEsScribeCfgV1
116+-- ES version 1 and defaulted to it.
117+defaultEsScribeCfg :: EsScribeCfg ESV5
118+defaultEsScribeCfg = defaultEsScribeCfgV5
113119
114120
115121-------------------------------------------------------------------------------
122+#if !MIN_VERSION_bloodhound(0,17,0)
116123-- | EsScribeCfg that will use ElasticSearch V1
117124defaultEsScribeCfgV1 :: EsScribeCfg ESV1
118125defaultEsScribeCfgV1 = defaultEsScribeCfg' (Typeable.Proxy :: Typeable.Proxy ESV1)
119−
126+#endif
120127
121128-------------------------------------------------------------------------------
122129-- | EsScribeCfg that will use ElasticSearch V5
@@ -240,7 +247,10 @@ splitTime t = asMins `divMod` 60
240247
241248-------------------------------------------------------------------------------
242249data EsScribeSetupError = CouldNotCreateIndex !(Response ByteString)
243− | CouldNotCreateMapping !(Response ByteString) deriving (Typeable, Show)
250+ | CouldNotUpdateIndexSettings !(Response ByteString)
251+ | CouldNotCreateMapping !(Response ByteString)
252+ | CouldNotPutTemplate !(Response ByteString)
253+ deriving (Typeable, Show)
244254
245255
246256instance Exception EsScribeSetupError
@@ -265,26 +275,34 @@ mkEsScribe
265275 -> IndexName v
266276 -- ^ Treated as a prefix if index sharding is enabled
267277 -> MappingName v
268− -> Severity
278+ -> PermitFunc
269279 -> Verbosity
270280 -> IO Scribe
271−mkEsScribe cfg@EsScribeCfg {..} env ix mapping sev verb = do
281+mkEsScribe cfg@EsScribeCfg {..} env ix mapping permit verb = do
272282 q <- newTBMQueueIO $ unEsQueueSize essQueueSize
273283 endSig <- newEmptyMVar
274284
275285 runBH prx env $ do
276− chk <- indexExists prx ix
277− -- note that this doesn't update settings. That's not available
278− -- through the Bloodhound API yet
279− unless chk $ void $ do
280− r1 <- createIndex prx essIndexSettings ix
281− unless (statusIsSuccessful (responseStatus r1)) $
282− liftIO $ throwIO (CouldNotCreateIndex r1)
283− r2 <- if shardingEnabled
284− then putTemplate prx tpl tplName
285− else putMapping prx ix mapping base
286− unless (statusIsSuccessful (responseStatus r2)) $
287− liftIO $ throwIO (CouldNotCreateMapping r2)
286+ if shardingEnabled
287+ then do
288+ -- create or update
289+ res <- putTemplate prx tpl tplName
290+ unless (statusIsSuccessful (responseStatus res)) $
291+ liftIO $ EX.throwIO (CouldNotPutTemplate res)
292+ else do
293+ ixExists <- indexExists prx ix
294+ if ixExists
295+ then do
296+ res <- updateIndexSettings prx (toUpdatabaleIndexSettings prx essIndexSettings) ix
297+ unless (statusIsSuccessful (responseStatus res)) $
298+ liftIO $ EX.throwIO (CouldNotUpdateIndexSettings res)
299+ else do
300+ r1 <- createIndex prx essIndexSettings ix
301+ unless (statusIsSuccessful (responseStatus r1)) $
302+ liftIO $ EX.throwIO (CouldNotCreateIndex r1)
303+ r2 <- putMapping prx ix mapping base
304+ unless (statusIsSuccessful (responseStatus r2)) $
305+ liftIO $ EX.throwIO (CouldNotCreateMapping r2)
288306
289307 workers <- replicateM (unEsPoolSize essPoolSize) $ async $
290308 startWorker cfg env mapping q
@@ -296,10 +314,10 @@ mkEsScribe cfg@EsScribeCfg {..} env ix mapping sev verb = do
296314 putMVar endSig ()
297315
298316 let finalizer = putMVar endSig () >> takeMVar endSig
299− return (Scribe (logger q) finalizer)
317+ return (Scribe (logger q) finalizer permit)
300318 where
301319 logger :: forall a. LogItem a => TBMQueue (IndexName v, Value) -> Item a -> IO ()
302− logger q i = when (_itemSeverity i >= sev) $
320+ logger q i =
303321 void $ atomically $ tryWriteTBMQueue q (chooseIxn prx ix essIndexSharding i, itemJson' i)
304322 prx :: Typeable.Proxy v
305323 prx = Typeable.Proxy
@@ -320,24 +338,27 @@ mkEsScribe cfg@EsScribeCfg {..} env ix mapping sev verb = do
320338baseMapping :: ESVersion v => proxy v -> MappingName v -> Value
321339baseMapping prx mn =
322340 object [ fromMappingName prx mn .= object ["properties" .= object prs] ]
323− where prs = [ str "thread"
324− , str "sev"
325− , str "pid"
326− , str "ns"
327− , str "msg"
341+ where prs = [ unanalyzedString "thread"
342+ , unanalyzedString "sev"
343+ , unanalyzedString "pid"
344+ -- ns is frequently fulltext searched
345+ , analyzedString "ns"
346+ -- we want message to be fulltext searchable
347+ , analyzedString "msg"
328348 , "loc" .= locType
329− , str "host"
330− , str "env"
349+ , unanalyzedString "host"
350+ , unanalyzedString "env"
331351 , "at" .= dateType
332− , str "app"
352+ , unanalyzedString "app"
333353 ]
334− str k = k .= object ["type" .= String "string"]
354+ unanalyzedString k = k .= unanalyzedStringSpec prx
355+ analyzedString k = k .= analyzedStringSpec prx
335356 locType = object ["properties" .= object locPairs]
336− locPairs = [ str "loc_pkg"
337− , str "loc_mod"
338− , str "loc_ln"
339− , str "loc_fn"
340− , str "loc_col"
357+ locPairs = [ unanalyzedString "loc_pkg"
358+ , unanalyzedString "loc_mod"
359+ , unanalyzedString "loc_ln"
360+ , unanalyzedString "loc_fn"
361+ , unanalyzedString "loc_col"
341362 ]
342363 dateType = object [ "format" .= esDateFormat
343364 , "type" .= String "date"
@@ -406,7 +427,7 @@ startWorker EsScribeCfg {..} env mapping q = go
406427 popped <- atomically $ readTBMQueue q
407428 case popped of
408429 Just (ixn, v) -> do
409− sendLog ixn v `catchAny` eat
430+ sendLog ixn v `EX.catchAny` eat
410431 go
411432 Nothing -> return ()
412433 prx :: Typeable.Proxy v
@@ -419,8 +440,8 @@ startWorker EsScribeCfg {..} env mapping q = go
419440 eat _ = return ()
420441 handler _ = Handler $ \e ->
421442 case fromException e of
422− Just (_ :: AsyncException) -> return False
423− _ -> return True
443+ Just (_ :: EX.SomeAsyncException) -> return False
444+ _ -> return True
424445
425446
426447-------------------------------------------------------------------------------
@@ -433,6 +454,7 @@ class ESVersion v where
433454 type BHEnv v
434455 type IndexSettings v
435456 defaultIndexSettings :: proxy v -> IndexSettings v
457+ type UpdatableIndexSetting v
436458 type IndexName v
437459 toIndexName :: proxy v -> Text -> IndexName v
438460 fromIndexName :: proxy v -> IndexName v -> Text
@@ -451,22 +473,35 @@ class ESVersion v where
451473 type IndexDocumentSettings v
452474 defaultIndexDocumentSettings :: proxy v -> IndexDocumentSettings v
453475
476+ toUpdatabaleIndexSettings :: proxy v -> IndexSettings v -> NonEmpty (UpdatableIndexSetting v)
477+
454478 -- Operations
455479 -- We're deciding on IO here, but it isn't necessary
456480 indexExists :: proxy v -> IndexName v -> BH v IO Bool
457481 indexDocument :: ToJSON doc => proxy v -> IndexName v -> MappingName v -> IndexDocumentSettings v -> doc -> DocId v -> BH v IO (Response ByteString)
458482 createIndex :: proxy v -> IndexSettings v -> IndexName v -> BH v IO (Response ByteString)
483+ updateIndexSettings :: proxy v -> NonEmpty (UpdatableIndexSetting v) -> IndexName v -> BH v IO (Response ByteString)
459484 putTemplate :: proxy v -> IndexTemplate v -> TemplateName v -> BH v IO (Response ByteString)
460485 putMapping :: (ToJSON a) => proxy v -> IndexName v -> MappingName v -> a -> BH v IO (Response ByteString)
461486
487+ -- In ES 5 and beyond, "string" was deprecated in favor of text for
488+ -- fulltext and keyword for unanalyzed tokens
489+ unanalyzedStringSpec :: proxy v -> Value
490+ analyzedStringSpec :: proxy v -> Value
491+
492+
462493
494+#if !MIN_VERSION_bloodhound(0,17,0)
463495data ESV1 = ESV1
496+{-# DEPRECATED ESV1 "ESV1 is deprecated and removed in bloodhound >= 0.17.0" #-}
497+
464498
465499instance ESVersion ESV1 where
466500 type BHEnv ESV1 = V1.BHEnv
467501 type IndexSettings ESV1 = V1.IndexSettings
468502 defaultIndexSettings _ = V1.defaultIndexSettings
469503 type IndexName ESV1 = V1.IndexName
504+ type UpdatableIndexSetting ESV1 = V1.UpdatableIndexSetting
470505 toIndexName _ = V1.IndexName
471506 fromIndexName _ (V1.IndexName x) = x
472507 type MappingName ESV1 = V1.MappingName
@@ -482,12 +517,24 @@ instance ESVersion ESV1 where
482517 type IndexTemplate ESV1 = V1.IndexTemplate
483518 toIndexTemplate _ = V1.IndexTemplate
484519 type IndexDocumentSettings ESV1 = V1.IndexDocumentSettings
520+ toUpdatabaleIndexSettings _ s =
521+ (V1.NumberOfReplicas (V1.indexReplicas s)) :| []
485522 defaultIndexDocumentSettings _ = V1.defaultIndexDocumentSettings
486523 indexExists _ = V1.indexExists
487524 indexDocument _ = V1.indexDocument
488525 createIndex _ = V1.createIndex
526+ updateIndexSettings _ = V1.updateIndexSettings
489527 putTemplate _ = V1.putTemplate
490528 putMapping _ = V1.putMapping
529+ unanalyzedStringSpec _ = object
530+ [ "type" .= String "string"
531+ , "index" .= String "not_analyzed"
532+ ]
533+ analyzedStringSpec _ = object
534+ [ "type" .= String "string"
535+ , "index" .= String "analyzed"
536+ ]
537+#endif
491538
492539
493540data ESV5 = ESV5
@@ -495,6 +542,7 @@ data ESV5 = ESV5
495542instance ESVersion ESV5 where
496543 type BHEnv ESV5 = V5.BHEnv
497544 type IndexSettings ESV5 = V5.IndexSettings
545+ type UpdatableIndexSetting ESV5 = V5.UpdatableIndexSetting
498546 defaultIndexSettings _ = V5.defaultIndexSettings
499547 type IndexName ESV5 = V5.IndexName
500548 toIndexName _ = V5.IndexName
@@ -512,9 +560,18 @@ instance ESVersion ESV5 where
512560 type IndexTemplate ESV5 = V5.IndexTemplate
513561 toIndexTemplate _ = V5.IndexTemplate
514562 type IndexDocumentSettings ESV5 = V5.IndexDocumentSettings
563+ toUpdatabaleIndexSettings _ s =
564+ (V5.NumberOfReplicas (V5.indexReplicas s)) :| []
515565 defaultIndexDocumentSettings _ = V5.defaultIndexDocumentSettings
516566 indexExists _ = V5.indexExists
517567 indexDocument _ = V5.indexDocument
518568 createIndex _ = V5.createIndex
569+ updateIndexSettings _ = V5.updateIndexSettings
519570 putTemplate _ = V5.putTemplate
520571 putMapping _ = V5.putMapping
572+ unanalyzedStringSpec _ = object
573+ [ "type" .= String "keyword"
574+ ]
575+ analyzedStringSpec _ = object
576+ [ "type" .= String "text"
577+ ]
katip-elasticsearch/test/Main.hs+35−10
@@ -17,6 +17,7 @@ module Main
1717-------------------------------------------------------------------------------
1818import Control.Applicative as A
1919import Control.Concurrent.STM
20+import Control.Exception
2021import Control.Lens hiding (mapping, (.=))
2122import Control.Monad
2223import Control.Monad.IO.Class
@@ -25,7 +26,7 @@ import Data.Aeson.Lens
2526import Data.Aeson.Types
2627import Data.ByteString.Lazy (ByteString)
2728import qualified Data.HashMap.Strict as HM
28−import Data.Monoid
29+import Data.Monoid as M
2930import Data.Scientific
3031import Data.Tagged
3132import Data.Text (Text)
@@ -33,8 +34,12 @@ import Data.Time
3334import Data.Time.Calendar.WeekDate
3435import Data.Typeable as Typeable
3536import qualified Data.Vector as V
37+#if MIN_VERSION_bloodhound(0,17,0)
38+import qualified Database.Bloodhound as V5
39+#else
3640import qualified Database.V1.Bloodhound as V1
3741import qualified Database.V5.Bloodhound as V5
42+#endif
3843import Network.HTTP.Client
3944import Network.HTTP.Types.Status
4045import Test.QuickCheck.Instances ()
@@ -53,8 +58,10 @@ main :: IO ()
5358main = defaultMainWithIngredients ings $ askOption $ \vers -> testGroup "katip-elasticsearch"
5459 [
5560 case vers of
56− TestV1 -> esTests (Typeable.Proxy :: Typeable.Proxy ESV1)
57− TestV5 -> esTests (Typeable.Proxy :: Typeable.Proxy ESV5)
61+#if !MIN_VERSION_bloodhound(0,17,0)
62+ TestV1 -> esTests (Typeable.Proxy :: Typeable.Proxy ESV1)
63+#endif
64+ TestV5 -> esTests (Typeable.Proxy :: Typeable.Proxy ESV5)
5865 , typeAnnotatedTests
5966 , roundToSundayTests
6067 ]
@@ -63,18 +70,26 @@ main = defaultMainWithIngredients ings $ askOption $ \vers -> testGroup "katip-e
6370
6471
6572-------------------------------------------------------------------------------
66−data TestWithESVersion = TestV1
67− | TestV5
68− deriving (Typeable)
73+data TestWithESVersion = TestV5
74+#if !MIN_VERSION_bloodhound(0,17,0)
75+ | TestV1
76+#endif
77+ deriving (Typeable)
6978
7079
7180instance IsOption TestWithESVersion where
72− defaultValue = TestV1
81+ defaultValue = TestV5
82+#if !MIN_VERSION_bloodhound(0,17,0)
7383 parseValue "1" = Just TestV1
84+#endif
7485 parseValue "5" = Just TestV5
7586 parseValue _ = Nothing
7687 optionName = Tagged "es-version"
77− optionHelp = Tagged "Version of ES to test against, either 1 or 5, defaulting to 1."
88+#if MIN_VERSION_bloodhound(0,17,0)
89+ optionHelp = Tagged "Version of ES to test against, either 1 or 5, defaulting to 5."
90+#else
91+ optionHelp = Tagged "Version of ES to test against, only v5 is supported, but should work for higher versions."
92+#endif
7893
7994
8095class ESVersion v => TestESVersion v where
@@ -100,6 +115,7 @@ class ESVersion v => TestESVersion v where
100115 searchByIndex :: proxy v -> IndexName v -> Search v -> BH v IO (Response ByteString)
101116
102117
118+#if !MIN_VERSION_bloodhound(0,17,0)
103119instance TestESVersion ESV1 where
104120 type Server ESV1 = V1.Server
105121 toServer _ = V1.Server
@@ -121,6 +137,7 @@ instance TestESVersion ESV1 where
121137 refreshIndex _ = V1.refreshIndex
122138 withBH _ = V1.withBH
123139 searchByIndex _ = V1.searchByIndex
140+#endif
124141
125142
126143instance TestESVersion ESV5 where
@@ -160,7 +177,7 @@ setupSearch
160177setupSearch prx modScribeCfg = do
161178 bh prx (dropESSchema prx)
162179 mgr <- newManager defaultManagerSettings
163− mkEsScribe cfg (mkBHEnv prx (svr prx) mgr) (ixn prx) (mn prx) DebugS V3
180+ mkEsScribe cfg (mkBHEnv prx (svr prx) mgr) (ixn prx) (mn prx) (permitItem DebugS) V3
164181 where
165182 cfg :: EsScribeCfg v
166183 cfg = modScribeCfg $
@@ -229,7 +246,7 @@ esTests
229246esTests prx = testGroup "elasticsearch scribe"
230247 [
231248 withSearch' (\c -> c { essIndexSharding = NoIndexSharding}) prx $ \setup -> testCase "it flushes to elasticsearch" $ withTestLogging prx setup $ \done -> do
232− $(logT) (ExampleCtx True) mempty InfoS "A test message"
249+ $(logT) (ExampleCtx True) M.mempty InfoS "A test message"
233250 liftIO $ do
234251 void done
235252 logs <- getLogs prx
@@ -237,6 +254,14 @@ esTests prx = testGroup "elasticsearch scribe"
237254 let l = head logs
238255 l ^? key "_source" . key "msg" . _String @?= Just "A test message"
239256 l ^? key "_source" . key "data" . key "whatever::b" . _Bool @?= Just True
257+ , testCase "can set up twice with no sharding" $ do
258+ let setup = bracket_ (setupSearch prx (\c -> c { essIndexSharding = NoIndexSharding })) (teardownSearch prx)
259+ setup $ setup $
260+ return ()
261+ , testCase "can set up twice with sharding" $ do
262+ let setup = bracket_ (setupSearch prx (\c -> c { essIndexSharding = DailyIndexSharding })) (teardownSearch prx)
263+ setup $ setup $
264+ return ()
240265 , withSearch prx $ \setup -> testCase "date-based index sharding" $ do
241266 let t1 = mkTime 2016 1 2 3 4 5
242267 fakeClock <- newTVarIO t1
katip-logzio/LICENSEnew file+30
@@ -0,0 +1,30 @@
1+Copyright (c) 2018, Soostone Inc
2+
3+All rights reserved.
4+
5+Redistribution and use in source and binary forms, with or without
6+modification, are permitted provided that the following conditions are met:
7+
8+ * Redistributions of source code must retain the above copyright
9+ notice, this list of conditions and the following disclaimer.
10+
11+ * Redistributions in binary form must reproduce the above
12+ copyright notice, this list of conditions and the following
13+ disclaimer in the documentation and/or other materials provided
14+ with the distribution.
15+
16+ * Neither the name of Ozgun Ataman nor the names of other
17+ contributors may be used to endorse or promote products derived
18+ from this software without specific prior written permission.
19+
20+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
21+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
22+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
23+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
24+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
25+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
26+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
27+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
28+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
29+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
30+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
katip-logzio/README.mdnew file+4
@@ -0,0 +1,4 @@
1+# Katip Logz.IO [![Build Status](https://travis-ci.org/Soostone/katip.svg?branch=master)](https://travis-ci.org/Soostone/katip)
2+
3+katip-logzio is a scribe for the Katip logging framework that sends
4+structured logs to the logz.io centralized logging platform.
katip-logzio/changelog.mdnew file+9
@@ -0,0 +1,9 @@
1+0.1.1.0
2+=======
3+* Fixed bug where on scribe close, any items left in the buffer that
4+ hadn't timed out or been flushed already would be dropped rather
5+ than flushed. All users are encouraged to upgrade.
6+
7+0.1.0.0
8+==========
9+* Initial release
katip-logzio/katip-logzio.cabalnew file+94
@@ -0,0 +1,94 @@
1+name: katip-logzio
2+version: 0.1.1.0
3+synopsis: Logz.IO scribe for the Katip logging framework
4+description: See README.md for more details.
5+license: BSD3
6+license-file: LICENSE
7+author: Michael Xavier
8+maintainer: michael.xavier@soostone.com
9+copyright: Soostone Inc, 2018
10+category: Data, Text, Logging
11+homepage: https://github.com/Soostone/katip
12+bug-reports: https://github.com/Soostone/katip/issues
13+build-type: Simple
14+cabal-version: >=1.10
15+extra-source-files:
16+ README.md
17+ changelog.md
18+
19+tested-with: GHC == 8.4.3
20+
21+source-repository head
22+ type: git
23+ location: https://github.com/Soostone/katip.git
24+
25+flag lib-Werror
26+ default: False
27+ manual: True
28+
29+
30+library
31+ exposed-modules:
32+ Katip.Scribes.LogzIO.HTTPS
33+ build-depends:
34+ base >=4.8.0.0 && <5
35+ , katip >= 0.8.0.0
36+ , stm >= 2.1.1
37+ , async
38+ , time >= 1.8.0.2
39+ , aeson >= 0.10.0.0
40+ , unix
41+ , unordered-containers
42+ , http-client >= 0.1.0
43+ , http-client-tls >= 0.3.4
44+ , uri-bytestring
45+ , errors >= 2.0.0
46+ , safe-exceptions
47+ , retry
48+ , http-types
49+ , text
50+ , bytestring
51+ , scientific
52+ , stm-chans >= 2.0.0
53+ hs-source-dirs: src
54+ default-language: Haskell2010
55+ hs-source-dirs: src
56+ ghc-options: -Wall -Wall -Wnoncanonical-monad-instances
57+ if flag(lib-Werror)
58+ ghc-options: -Werror
59+
60+
61+test-suite test
62+ type: exitcode-stdio-1.0
63+ main-is: Main.hs
64+ hs-source-dirs: test
65+ default-language: Haskell2010
66+ other-modules:
67+ Katip.Tests.Scribes.LogzIO.HTTPS
68+ build-depends: base
69+ , katip
70+ , katip-logzio
71+ , tasty
72+ , tasty-hedgehog
73+ , tasty-hunit
74+ , hedgehog
75+ , vector
76+ , unordered-containers
77+ , text
78+ , bytestring
79+ , time
80+ , template-haskell
81+ , aeson
82+ , scientific
83+ , unix
84+ , hostname
85+ , scotty
86+ , stm
87+ , async
88+ , safe-exceptions
89+ , uri-bytestring
90+ , warp
91+ , http-types
92+ ghc-options: -Wall -threaded -rtsopts -with-rtsopts=-N -Wnoncanonical-monad-instances
93+ if flag(lib-Werror)
94+ ghc-options: -Werror
katip-logzio/src/Katip/Scribes/LogzIO/HTTPS.hsnew file+639
@@ -0,0 +1,639 @@
1+{-# LANGUAGE BangPatterns #-}
2+{-# LANGUAGE GADTs #-}
3+{-# LANGUAGE GeneralizedNewtypeDeriving #-}
4+{-# LANGUAGE OverloadedStrings #-}
5+{-# LANGUAGE RankNTypes #-}
6+{-# LANGUAGE TypeApplications #-}
7+-- | This is a log scribe that writes logs to logz.io's bulk
8+-- <https://app.logz.io/#/dashboard/data-sources/Bulk-HTTPS HTTPS
9+-- API>.
10+module Katip.Scribes.LogzIO.HTTPS
11+ ( -- * Scribe construction
12+ mkLogzIOScribe
13+ -- * Types
14+ , BulkAPIError(..)
15+ , LogzIOScribeConfiguration(..)
16+ , Scheme(..)
17+ , APIToken(..)
18+ , LoggingError(..)
19+ -- ** Presets for configuration
20+ , usRegionHost
21+ , euRegionHost
22+ , httpsPort
23+ , httpPort
24+ , defaultRetryPolicy
25+ , defaultLogzIOScribeConfiguration
26+ -- * Internal API exported for testing
27+ , renderLineTruncated
28+ , renderLineTruncated'
29+ , maxPayloadBytes
30+ , maxLogLineLength
31+ , BulkBuffer (..)
32+ , LogAction (..)
33+ , bufferItem
34+ , bufferItem'
35+ , forceFlush
36+ , Bytes (..)
37+ ) where
38+
39+
40+-------------------------------------------------------------------------------
41+import Control.Applicative
42+import qualified Control.Concurrent.Async as Async
43+import qualified Control.Concurrent.STM as STM
44+import qualified Control.Concurrent.STM.TBMQueue as STM
45+import qualified Control.Error as E
46+import qualified Control.Exception.Safe as EX
47+import Control.Monad
48+import qualified Control.Retry as Retry
49+import qualified Data.Aeson as A
50+import qualified Data.ByteString.Builder as BB
51+import qualified Data.ByteString.Lazy as LBS
52+import qualified Data.ByteString.Lazy.Char8 as LBS8
53+import qualified Data.HashMap.Strict as HM
54+import Data.Int
55+import qualified Data.Scientific as Scientific
56+import Data.Semigroup as Semigroup
57+import Data.String (IsString)
58+import qualified Data.Text as T
59+import qualified Data.Text.Encoding as TE
60+import qualified Data.Text.Lazy as TL
61+import qualified Data.Text.Lazy.Builder as TB
62+import qualified Data.Time as Time
63+import qualified Katip as K
64+import Katip.Core (LocJs (..))
65+import qualified Network.HTTP.Client as HTTP
66+import qualified Network.HTTP.Client.TLS as HTTPS
67+import qualified Network.HTTP.Types as HTypes
68+import qualified System.Posix.Types as POSIX
69+import qualified URI.ByteString as URIBS
70+-------------------------------------------------------------------------------
71+
72+
73+
74+-- | This is returned when the bulk import was a partial success
75+data BulkAPIError = BulkAPIError
76+ { bulkAPIError_malformedLines :: Int
77+ -- ^ The number of log lines which are not well-formed JSON. This
78+ -- indicates a __library bug__. Please file an issue on GitHub.
79+ , bulkAPIError_successfulLines :: Int
80+ -- ^ The number of log lines received successfully.
81+ , bulkAPIError_oversizedLines :: Int
82+ -- ^ The number of log lines which exceed the line length
83+ -- limit. katip-logzio makes a best effort to truncate logs that
84+ -- exceed the size limit. This probably indicates a __library bug__
85+ -- and should be reported as an issue on GitHub.
86+ , bulkAPIError_emptyLogLines :: Int
87+ -- ^ The number of log lines which were empty. There isnt' really a
88+ -- concept of a truly empty log line in katip, so this most likely
89+ -- indicates a __library bug__ and should be reported as an issue on
90+ -- GitHub.
91+ } deriving (Show, Eq)
92+
93+
94+instance A.FromJSON BulkAPIError where
95+ parseJSON = A.withObject "BulkAPIError" $ \o -> do
96+ malformedLines <- o A..: "malformedLines"
97+ successfulLines <- o A..: "successfulLines"
98+ oversizedLines <- o A..: "oversizedLines"
99+ emptyLogLines <- o A..: "emptyLogLines"
100+ pure $ BulkAPIError
101+ { bulkAPIError_malformedLines = malformedLines
102+ , bulkAPIError_successfulLines = successfulLines
103+ , bulkAPIError_oversizedLines = oversizedLines
104+ , bulkAPIError_emptyLogLines = emptyLogLines
105+ }
106+
107+
108+-------------------------------------------------------------------------------
109+data LogzIOScribeConfiguration = LogzIOScribeConfiguration
110+ { logzIOScribeConfiguration_bufferItems :: Int
111+ -- ^ Will flush the log buffer if this many items is in the buffer
112+ -- __or__ 'logzIOScribeConfiguration_bufferTimeout' is reached,
113+ -- whichever is first
114+ , logzIOScribeConfiguration_bufferTimeout :: Time.NominalDiffTime
115+ -- ^ Will flush the buffer if it has been this long since the last
116+ -- flush __or__ 'logzIOScribeConfiguration_bufferItems' items are
117+ -- accumulated, whichever is first. NominalDiffTime has a Num
118+ -- instance, so you can use a literal to specify seconds, e.g. 30 =
119+ -- 30s.
120+ , logzIOScribeConfiguration_scheme :: Scheme
121+ , logzIOScribeConfiguration_host :: URIBS.Host
122+ , logzIOScribeConfiguration_port :: URIBS.Port
123+ , logzIOScribeConfiguration_token :: APIToken
124+ , logzIOScribeConfiguration_retry :: Retry.RetryPolicyM IO
125+ -- ^ How should exceptions during writes be retried?
126+ , logzIOScribeConfiguration_onError :: LoggingError -> IO ()
127+ }
128+
129+
130+-- | A default configuration:
131+--
132+-- * 100 item buffering
133+--
134+-- * 30 second buffer timeout
135+--
136+-- * US, HTTPS logging endpoint (listener.logz.io:8071)
137+--
138+-- * 'defaultRetryPolicy' of 25ms exponential backoff, 5 retries
139+--
140+-- * Ignore logging errors
141+defaultLogzIOScribeConfiguration :: APIToken -> LogzIOScribeConfiguration
142+defaultLogzIOScribeConfiguration token = LogzIOScribeConfiguration
143+ { logzIOScribeConfiguration_bufferItems = 100
144+ , logzIOScribeConfiguration_bufferTimeout = 30
145+ , logzIOScribeConfiguration_scheme = HTTPS
146+ , logzIOScribeConfiguration_host = usRegionHost
147+ , logzIOScribeConfiguration_port = httpsPort
148+ , logzIOScribeConfiguration_token = token
149+ , logzIOScribeConfiguration_retry = defaultRetryPolicy
150+ , logzIOScribeConfiguration_onError = const (pure ())
151+ }
152+
153+
154+-------------------------------------------------------------------------------
155+-- | You can retrieve your account or sub-account's API token on the
156+-- <https://app.logz.io/#/dashboard/settings/manage-accounts manage
157+-- accounts page>. Note that APIToken has an IsString instance,
158+-- meaning that you can use a string literal with OverloadedStrings
159+-- enabled.
160+newtype APIToken = APIToken
161+ { apiToken :: T.Text
162+ } deriving (Show, Eq, IsString)
163+
164+
165+-- | This particular bulk API only supports HTTP or HTTPS. HTTPS is
166+-- strongly recommended for security reasons.
167+data Scheme
168+ = HTTPS
169+ -- ^ HTTPs should always be used except for local testing.
170+ | HTTP
171+ deriving (Show, Eq)
172+
173+
174+-- | See
175+-- <https://support.logz.io/hc/en-us/articles/210206365-What-IP-addresses-should-I-open-in-my-firewall-to-ship-logs-to-Logz-io-
176+-- this> for a list of listeners. This is the US region host,
177+-- listener.logz.io
178+usRegionHost :: URIBS.Host
179+usRegionHost = URIBS.Host "listener.logz.io"
180+
181+
182+-- | See
183+-- <https://support.logz.io/hc/en-us/articles/210206365-What-IP-addresses-should-I-open-in-my-firewall-to-ship-logs-to-Logz-io-
184+-- this> for a list of listeners. This is the EU region host,
185+-- listener.logz.io
186+euRegionHost :: URIBS.Host
187+euRegionHost = URIBS.Host "listener-eu.logz.io"
188+
189+
190+-- | Logz.io uses port 8071 for HTTPS
191+httpsPort :: URIBS.Port
192+httpsPort = URIBS.Port 8071
193+
194+
195+-- | Logz.io uses port 8070 for HTTP
196+httpPort :: URIBS.Port
197+httpPort = URIBS.Port 8070
198+
199+
200+-- | A reasonable retry policy: exponential backoff with 25ms base
201+-- delay up to 5 retries, for a total cumulative delay of 775ms.
202+defaultRetryPolicy :: (Monad m) => Retry.RetryPolicyM m
203+defaultRetryPolicy = Retry.exponentialBackoff 25000 `mappend` Retry.limitRetries 5
204+
205+-------------------------------------------------------------------------------
206+mkLogzIOScribe
207+ :: LogzIOScribeConfiguration
208+ -> K.PermitFunc
209+ -> K.Verbosity
210+ -> IO K.Scribe
211+mkLogzIOScribe config permitItem verbosity = do
212+ -- This size is actually somewhat arbitrary. We're just making sure
213+ -- it isn't infinite so we don't consume the whole backlog right
214+ -- away and take up lots of memory.. Katip above us does bounded
215+ -- buffering and load-shedding when there's too much of a
216+ -- backlog. Our writes are blocking to apply backpressure up to
217+ -- katip. This value just holds onto values temporarily while we
218+ -- concatenate them into a buffer.
219+ ingestionQueue <- STM.newTBMQueueIO (itemBufferSize * 10)
220+
221+ let newTimer = STM.registerDelay itemBufferTimeoutMicros
222+ timerRef <- STM.newTVarIO =<< newTimer
223+
224+ -- Set up a connection manager for requests
225+ mgr <- case scheme of
226+ HTTPS -> HTTPS.newTlsManager
227+ HTTP -> HTTP.newManager HTTP.defaultManagerSettings
228+
229+ -- An STM transaction that will return true when writes are stopped
230+ -- and backlog is emptied.
231+ let workExhausted :: STM.STM Bool
232+ workExhausted = (&&)
233+ <$> STM.isClosedTBMQueue ingestionQueue
234+ <*> STM.isEmptyTBMQueue ingestionQueue
235+ -- Block until there's no more work
236+ let waitWorkExhausted :: STM.STM ()
237+ waitWorkExhausted = STM.check =<< workExhausted
238+ let pop :: STM.STM (Tick AnyLogItem)
239+ pop = maybe WorkExhausted NewItem <$> STM.readTBMQueue ingestionQueue
240+
241+ let timeExpired :: STM.STM (Tick a)
242+ timeExpired = do
243+ isExpired <- STM.readTVar =<< STM.readTVar timerRef
244+ STM.check isExpired
245+ pure TimeExpired
246+
247+ -- Circular transaction that checks for completion of work, time
248+ -- expiration, or new events.
249+ let nextTick :: STM.STM (Tick AnyLogItem)
250+ nextTick =
251+ timeExpired <|>
252+ pop <|>
253+ (WorkExhausted <$ waitWorkExhausted)
254+
255+ -- Blocking push. This applies backpressure upstream to katip, which does its own buffering
256+ let push :: AnyLogItem -> STM.STM ()
257+ push = void . STM.writeTBMQueue ingestionQueue
258+
259+ let sealQueue = STM.atomically (STM.closeTBMQueue ingestionQueue)
260+
261+ -- Replace the timer with a new one
262+ let resetTimer = STM.atomically . STM.writeTVar timerRef =<< newTimer
263+
264+ -- Send the buffer and then reset the timer
265+ let flush curBuffer = do
266+ res <- flushBuffer config mgr curBuffer
267+ case res of
268+ Left e -> onErrorSafe e
269+ Right () -> pure ()
270+ resetTimer
271+
272+ let flushLoop :: BulkBuffer -> IO ()
273+ flushLoop curBuffer = do
274+ tick <- STM.atomically nextTick
275+ case tick of
276+ WorkExhausted -> do
277+ flush curBuffer -- flush what you've got
278+ pure () -- stop looping
279+ TimeExpired -> do
280+ flush curBuffer
281+ flushLoop mempty
282+ NewItem (AnyLogItem item) -> do
283+ case bufferItem (logzIOScribeConfiguration_bufferItems config) verbosity item curBuffer of
284+ Buffered newBuffer -> flushLoop newBuffer
285+ FlushNow flushThis newBuffer -> do
286+ flush flushThis
287+ flushLoop newBuffer
288+
289+ flushThread <- Async.async (flushLoop mempty)
290+
291+ let close = do
292+ sealQueue
293+ _ <- Async.waitCatch flushThread
294+ pure ()
295+
296+ pure $ K.Scribe
297+ { K.liPush = STM.atomically . push . AnyLogItem
298+ , K.scribeFinalizer = close
299+ , K.scribePermitItem = permitItem
300+ }
301+ where
302+ itemBufferSize = logzIOScribeConfiguration_bufferItems config
303+ itemBufferTimeoutMicros = ndtToMicros (logzIOScribeConfiguration_bufferTimeout config)
304+ scheme = logzIOScribeConfiguration_scheme config
305+ onErrorSafe ex = do
306+ _ <- EX.tryAny (logzIOScribeConfiguration_onError config ex)
307+ pure ()
308+
309+
310+data AnyLogItem where
311+ AnyLogItem :: K.LogItem a => K.Item a -> AnyLogItem
312+
313+
314+-------------------------------------------------------------------------------
315+data Tick a =
316+ TimeExpired
317+ | NewItem !a
318+ | WorkExhausted
319+
320+
321+-------------------------------------------------------------------------------
322+-- Match the native resolution of NominalDiffTime, which is excessive
323+-- for our needs
324+ndtPicos :: Time.NominalDiffTime -> Int64
325+ndtPicos = round . (* picos)
326+ where
327+ picos :: Time.NominalDiffTime
328+ picos = 10 ^ (9 :: Int)
329+
330+
331+-------------------------------------------------------------------------------
332+ndtToMicros :: Time.NominalDiffTime -> Int
333+ndtToMicros t = round (((fromIntegral (ndtPicos t)) :: Double) / picosInMicro)
334+ where
335+ picosInMicro = 10 ^ (3 :: Int)
336+
337+
338+-------------------------------------------------------------------------------
339+data LoggingError =
340+ URIError HTTP.HttpException
341+ -- ^ The URI generated was invalid. Check your configuration
342+ | RequestError HTTP.HttpException
343+ -- ^ We encountered an exception while sending the batch request
344+ | PartialFailure BulkAPIError
345+ -- ^ Some or all of the request was rejected. Check the logz.io UI
346+ -- for indexing errors.
347+ | BadToken
348+ -- ^ Your API token was rejected.
349+ | UnknownFailureResponse HTypes.Status LBS.ByteString
350+ -- ^ An error returned, but it could not be decoded into a
351+ -- 'BulkAPIError'. This may indicate a __library bug__, which should
352+ -- be reported to the issue tracker.
353+ deriving (Show)
354+
355+-------------------------------------------------------------------------------
356+flushBuffer
357+ :: LogzIOScribeConfiguration
358+ -> HTTP.Manager
359+ -> BulkBuffer
360+ -> IO (Either LoggingError ())
361+flushBuffer config mgr bulkBuffer
362+ | bulkBuffer_itemCount bulkBuffer <= 0 = pure (Right ())
363+ | otherwise = do
364+ E.runExceptT $ do
365+ req <- E.fmapLT URIError (E.ExceptT (EX.try (configureRequest <$> HTTP.parseRequest uriStr)))
366+ resp <- E.fmapLT RequestError $ E.ExceptT $ EX.try $
367+ Retry.recovering retryPolicy [\_stat -> EX.Handler handleHttpException] $ \_stat ->
368+ HTTP.httpLbs req mgr
369+ let respLBS = HTTP.responseBody resp
370+ let respStatus = HTTP.responseStatus resp
371+ if HTypes.statusIsSuccessful respStatus
372+ then pure ()
373+ else case A.decode @BulkAPIError respLBS of
374+ Nothing
375+ | HTypes.statusCode respStatus == 401 -> E.throwE BadToken
376+ | otherwise -> E.throwE (UnknownFailureResponse respStatus respLBS)
377+ Just bulkError -> E.throwE (PartialFailure bulkError)
378+ where
379+ configureRequest req = req
380+ { HTTP.method = HTypes.methodPost
381+ , HTTP.requestBody = HTTP.RequestBodyLBS (BB.toLazyByteString (bulkBuffer_payload bulkBuffer))
382+ }
383+
384+ retryPolicy = logzIOScribeConfiguration_retry config
385+ apiTokenBS = TE.encodeUtf8 (apiToken (logzIOScribeConfiguration_token config))
386+
387+ handleHttpException :: (Applicative m) => HTTP.HttpException -> m Bool
388+ handleHttpException _ = pure True
389+
390+ uriStr = LBS8.unpack (BB.toLazyByteString (URIBS.serializeURIRef uri))
391+
392+ authority = URIBS.Authority
393+ { URIBS.authorityUserInfo = Nothing
394+ , URIBS.authorityHost = logzIOScribeConfiguration_host config
395+ , URIBS.authorityPort = Just (logzIOScribeConfiguration_port config)
396+ }
397+
398+ uri = URIBS.URI
399+ { URIBS.uriScheme = case logzIOScribeConfiguration_scheme config of
400+ HTTPS -> URIBS.Scheme "https"
401+ HTTP -> URIBS.Scheme "http"
402+ , URIBS.uriAuthority = Just authority
403+ , URIBS.uriPath = "/"
404+ , URIBS.uriQuery = URIBS.Query
405+ [ ("token", apiTokenBS)
406+ ]
407+ , URIBS.uriFragment = Nothing
408+ }
409+
410+
411+-------------------------------------------------------------------------------
412+newtype Bytes = Bytes
413+ { bytes :: Int64
414+ } deriving (Show, Eq, Num, Ord, Bounded)
415+
416+
417+-- | How big of a body can we send? The limit is defined
418+-- <https://app.logz.io/#/dashboard/data-sources/Bulk-HTTPS here>.
419+maxPayloadBytes :: Bytes
420+maxPayloadBytes = 10485760
421+
422+
423+-- | How long can each serialized payload be (let's assume including
424+-- the trailing newline). The limit is defined
425+-- <https://app.logz.io/#/dashboard/data-sources/Bulk-HTTPS here>.
426+maxLogLineLength :: Bytes
427+maxLogLineLength = 500000
428+
429+
430+measureJSONLine :: A.ToJSON a => a -> (BB.Builder, Bytes)
431+measureJSONLine a = (BB.lazyByteString lbs, Bytes (LBS.length lbs))
432+ where
433+ lbs = A.encode a <> "\n"
434+
435+
436+-- | Fully-rendered JSON object for an item
437+fullItemObject :: K.LogItem a => K.Verbosity -> K.Item a -> A.Object
438+fullItemObject verbosity item = HM.fromList
439+ [ "app" A..= K._itemApp item
440+ , "env" A..= K._itemEnv item
441+ , "sev" A..= K._itemSeverity item
442+ , "thread" A..= K.getThreadIdText (K._itemThread item)
443+ , "host" A..= K._itemHost item
444+ , "pid" A..= pidInt
445+ , "data" A..= annotateKeys (K.payloadObject verbosity (K._itemPayload item))
446+ -- Slight deviation, logz.io uses "message" instead of "msg"
447+ , "message" A..= TB.toLazyText (K.unLogStr (K._itemMessage item))
448+ -- Another slight deviation, logz.io uses "@timestamp" instead of
449+ -- "at". Note, your logs should be sent roughly close to when they
450+ -- are created. They are assigned to indexes based on index date, so
451+ -- time searches act strange if you backfill too far from the
452+ -- past. They seem to support 3 decimal places of
453+ -- precision. Formatting like this requires time 1.8.0.2, which is
454+ -- reflected in the cabal file.
455+ , "@timestamp" A..= A.String (T.pack (Time.formatTime Time.defaultTimeLocale "%Y-%m-%dT%H:%M:%S%03QZ" (K._itemTime item)))
456+ , "ns" A..= K._itemNamespace item
457+ , "loc" A..= (LocJs <$> K._itemLoc item)
458+ ]
459+ where
460+ POSIX.CPid pidInt = K._itemProcess item
461+
462+
463+-- | A version of 'renderLine' which renders a line and stays under
464+-- the maximum line size of 500,000 bytes. If the default rendering is
465+-- too large, the log will be reduced to a timestamp and a potentially
466+-- truncated message.
467+renderLineTruncated :: K.LogItem a => K.Verbosity -> K.Item a -> (BB.Builder, Bytes)
468+renderLineTruncated = renderLineTruncated' maxLogLineLength
469+
470+
471+-- | A generalized renderLineTruncated that takes a custom line length
472+-- limit. This is exclusively for testing.
473+renderLineTruncated'
474+ :: K.LogItem a
475+ => Bytes
476+ -- ^ Custom max log line length. Be careful, too low and no amount
477+ -- of shrinking can get under the size, breaking the invariant. For
478+ -- the production limit, we are guraanteed always able to come in
479+ -- under the limit.
480+ -> K.Verbosity
481+ -> K.Item a
482+ -> (BB.Builder, Bytes)
483+renderLineTruncated' customMaxLogLineLength verbosity item =
484+ if fullSize <= customMaxLogLineLength
485+ then (fullLine, fullSize)
486+ else (fallbackLine, fallbackSize)
487+ where
488+ fullObject = fullItemObject verbosity item
489+ (fullLine, fullSize) = measureJSONLine fullObject
490+ -- only the absolutely necessary keys, with message stripped out
491+ -- to help us calculate how much of the message we can keep. these
492+ -- are lazily evaluated and as such won't be computed unless the
493+ -- item is too big
494+ blankObject :: A.Object
495+ blankObject = HM.fromList
496+ [ "message" A..= A.String "" -- we'll start with a blank message
497+ , "@timestamp" A..= K._itemTime item
498+ ]
499+ (_, blankObjectSize) = measureJSONLine blankObject
500+ messageBytesAllowed = maxLogLineLength - blankObjectSize
501+ (fallbackLine, fallbackSize) = measureJSONLine fallbackObject
502+ fallbackObject :: A.Object
503+ fallbackObject = HM.fromList
504+ [ "message" A..= A.toJSON (TL.take (bytes messageBytesAllowed) (TB.toLazyText (K.unLogStr (K._itemMessage item))))
505+ , "@timestamp" A..= A.toJSON (K._itemTime item)
506+ ]
507+
508+
509+data BulkBuffer = BulkBuffer
510+ { bulkBuffer_bytesUsed :: !Bytes
511+ , bulkBuffer_payload :: !BB.Builder
512+ , bulkBuffer_itemCount :: !Int
513+ }
514+
515+
516+instance Semigroup.Semigroup BulkBuffer where
517+ (BulkBuffer bytesUsedA bufferA itemCountA) <>
518+ (BulkBuffer bytesUsedB bufferB itemCountB) = BulkBuffer
519+ (bytesUsedA + bytesUsedB)
520+ (bufferA <> bufferB)
521+ (itemCountA + itemCountB)
522+
523+
524+instance Monoid BulkBuffer where
525+ mempty = BulkBuffer 0 mempty 0
526+ mappend = (<>)
527+
528+
529+data LogAction buf =
530+ Buffered
531+ buf
532+ -- ^ New buffer
533+ | FlushNow
534+ buf
535+ -- ^ Buffer to flush
536+ buf
537+ -- ^ New buffer
538+
539+bufferItem
540+ :: K.LogItem a
541+ => Int
542+ -- ^ Maximum items before flushing
543+ -> K.Verbosity
544+ -> K.Item a
545+ -> BulkBuffer
546+ -> LogAction BulkBuffer
547+bufferItem = bufferItem' maxPayloadBytes maxLogLineLength
548+
549+
550+-- | An internal version with a configurable max item size and max
551+-- message size for testing. Be careful with this: if you set the
552+-- values too low, some of the guarantees about reducability of
553+-- messages break down.
554+bufferItem'
555+ :: (K.LogItem a)
556+ => Bytes
557+ -- ^ Max payload size
558+ -> Bytes
559+ -- ^ Max item size
560+ -> Int
561+ -- ^ Maximum items before flushing
562+ -> K.Verbosity
563+ -> K.Item a
564+ -> BulkBuffer
565+ -> LogAction BulkBuffer
566+bufferItem' customMaxPayload customMaxItem maxItems verb item bulkBuffer =
567+ let (encodedLine, spaceNeeded) = renderLineTruncated' customMaxItem verb item
568+ newBytesUsed = bulkBuffer_bytesUsed bulkBuffer + spaceNeeded
569+ newItemCount = bulkBuffer_itemCount bulkBuffer + 1
570+ in if newItemCount >= maxItems || newBytesUsed >= customMaxPayload
571+ then FlushNow
572+ bulkBuffer
573+ BulkBuffer
574+ { bulkBuffer_bytesUsed = spaceNeeded
575+ , bulkBuffer_payload = encodedLine
576+ , bulkBuffer_itemCount = 1
577+ }
578+ else Buffered $
579+ BulkBuffer
580+ { bulkBuffer_bytesUsed = newBytesUsed
581+ , bulkBuffer_payload = bulkBuffer_payload bulkBuffer <> encodedLine
582+ , bulkBuffer_itemCount = newItemCount
583+ }
584+
585+
586+-- | When time has run out on a flush, splits the buffer. n.b. this
587+-- will always return 'FlushNow' with an empty new buffer.
588+forceFlush :: (Monoid buf) => buf -> LogAction buf
589+forceFlush buf = FlushNow buf mempty
590+
591+
592+-------------------------------------------------------------------------------
593+-- Annotation borrowed from katip-elasticsearch. There are fixed
594+-- fields in katip logs which should stay the same type forever and
595+-- thus won't need annotation. However, any code in userland may
596+-- choose to add akey to their log's metadata with a type that's
597+-- incompatible with the mapping. If the first value that's picked up
598+-- restrictive (e.g. a long), subsequent values will be rejected. By
599+-- differentiating types by their name, this is guaranteed not to
600+-- happen.
601+
602+annotateValue :: A.Value -> A.Value
603+annotateValue (A.Object o) = A.Object (annotateKeys o)
604+annotateValue (A.Array a) = A.Array (annotateValue <$> a)
605+annotateValue x = x
606+
607+
608+annotateKeys :: A.Object -> A.Object
609+annotateKeys = HM.fromList . map go . HM.toList
610+ where
611+ go (k, A.Object o) = (k, A.Object (annotateKeys o))
612+ go (k, A.Array a) = (k, A.Array (annotateValue <$> a))
613+ go (k, s@(A.String _)) = (k <> stringAnn, s)
614+ go (k, n@(A.Number sci)) = if Scientific.isFloating sci
615+ then (k <> doubleAnn, n)
616+ else (k <> longAnn, n)
617+ go (k, b@(A.Bool _)) = (k <> booleanAnn, b)
618+ go (k, A.Null) = (k <> nullAnn, A.Null)
619+
620+
621+-------------------------------------------------------------------------------
622+-- Annotation Constants
623+-------------------------------------------------------------------------------
624+
625+
626+stringAnn :: T.Text
627+stringAnn = "::s"
628+
629+doubleAnn :: T.Text
630+doubleAnn = "::d"
631+
632+longAnn :: T.Text
633+longAnn = "::l"
634+
635+booleanAnn :: T.Text
636+booleanAnn = "::b"
637+
638+nullAnn :: T.Text
639+nullAnn = "::n"
katip-logzio/test/Katip/Tests/Scribes/LogzIO/HTTPS.hsnew file+466
@@ -0,0 +1,466 @@
1+{-# LANGUAGE OverloadedStrings #-}
2+module Katip.Tests.Scribes.LogzIO.HTTPS
3+ ( tests
4+ ) where
5+
6+
7+-------------------------------------------------------------------------------
8+import Control.Concurrent.Async
9+import Control.Concurrent.STM
10+import Control.Concurrent.STM.TSem
11+import Control.Exception.Safe
12+import Control.Monad
13+import Control.Monad.IO.Class
14+import Data.Aeson as A
15+import qualified Data.ByteString.Builder as BB
16+import qualified Data.ByteString.Lazy as LBS
17+import qualified Data.ByteString.Lazy.Char8 as LBS8
18+import qualified Data.HashMap.Strict as HM
19+import Data.Int
20+import Data.Scientific
21+import Data.Text (Text)
22+import Data.Time
23+import Data.Time.Clock.POSIX
24+import qualified Data.Vector as V
25+import Hedgehog as HH
26+import qualified Hedgehog.Gen as Gen
27+import qualified Hedgehog.Range as Range
28+import Katip
29+import Katip.Core
30+import Language.Haskell.TH (Loc (..))
31+import Network.HostName
32+import qualified Network.HTTP.Types.Status as HTypes
33+import qualified Network.Wai.Handler.Warp as Warp
34+import System.Posix
35+import System.Timeout
36+import Test.Tasty
37+import Test.Tasty.Hedgehog
38+import Test.Tasty.HUnit
39+import URI.ByteString
40+import qualified Web.Scotty as Scotty
41+-------------------------------------------------------------------------------
42+import Katip.Scribes.LogzIO.HTTPS
43+-------------------------------------------------------------------------------
44+
45+
46+tests :: TestTree
47+tests = testGroup "Katip.Scribes.LogzIO.HTTP"
48+ [ bufferItemTests
49+ , renderLineTruncatedTests
50+ , scribeTests
51+ ]
52+
53+
54+-------------------------------------------------------------------------------
55+-- We bind to a system port so we use a lock on these tests to avoid interleaved, concurrent requests
56+scribeTests :: TestTree
57+scribeTests = withLock $ \mkSem -> testGroup "scribe"
58+ [ testCase "writes well-formed log items as lines to the given API" $ do
59+ sem <- mkSem
60+ withServer sem $ \receivedMessages -> do
61+ withScribe (\cfg -> cfg { logzIOScribeConfiguration_bufferTimeout = 1}) $ \scribe -> do
62+ (liPush scribe) (genericLog "a message")
63+
64+ requestLog <- timeoutFailure (20 * second) (atomically (readTQueue receivedMessages))
65+ requestLog_token requestLog @?= testAPIToken
66+ let logs = requestLog_logs requestLog
67+ length logs @?= 1
68+ let firstLog = head logs
69+ receivedLog_message firstLog @?= "a message"
70+ , testCase "respects the items limit" $ do
71+ sem <- mkSem
72+ withServer sem $ \receivedMessages -> do
73+ withScribe (\cfg -> cfg { logzIOScribeConfiguration_bufferItems = 1, logzIOScribeConfiguration_bufferTimeout = 1}) $ \scribe -> do
74+ (liPush scribe) (genericLog "message 1")
75+ (liPush scribe) (genericLog "message 2")
76+
77+ log1 <- timeoutFailure (20 * second) (atomically (readTQueue receivedMessages))
78+ requestLog_token log1 @?= testAPIToken
79+ let logs1 = requestLog_logs log1
80+ length logs1 @?= 1
81+ let firstLog1 = head logs1
82+ receivedLog_message firstLog1 @?= "message 1"
83+
84+ log2 <- timeoutFailure (20 * second) (atomically (readTQueue receivedMessages))
85+ let logs2 = requestLog_logs log2
86+ length logs2 @?= 1
87+ let firstLog2 = head logs2
88+ receivedLog_message firstLog2 @?= "message 2"
89+ , testCase "respects the time limit" $ do
90+ sem <- mkSem
91+ withServer sem $ \receivedMessages -> do
92+ withScribe (\cfg -> cfg { logzIOScribeConfiguration_bufferItems = 2, logzIOScribeConfiguration_bufferTimeout = 1}) $ \scribe -> do
93+ (liPush scribe) (genericLog "a message")
94+ log1 <- timeoutFailure (20 * second) (atomically (readTQueue receivedMessages))
95+ requestLog_token log1 @?= testAPIToken
96+ let logs1 = requestLog_logs log1
97+ length logs1 @?= 1
98+ let firstLog1 = head logs1
99+ receivedLog_message firstLog1 @?= "a message"
100+ , testCase "reports errors" $ do
101+ sem <- mkSem
102+ let badToken = APIToken "badtoken"
103+ errorSignal <- newEmptyTMVarIO
104+ withServer sem $ \_receivedMessages -> do
105+ withScribe (\cfg -> cfg { logzIOScribeConfiguration_bufferItems = 1, logzIOScribeConfiguration_bufferTimeout = 1, logzIOScribeConfiguration_token = badToken, logzIOScribeConfiguration_onError = void . atomically . tryPutTMVar errorSignal}) $ \scribe -> do
106+ (liPush scribe) (genericLog "a message")
107+ loggingError <- timeoutFailure (20 * second) (atomically (takeTMVar errorSignal))
108+ case loggingError of
109+ BadToken -> pure ()
110+ _ -> assertFailure ("Expected BadToken error but got " <> show loggingError)
111+ , testCase "flushes all queued logs on close" $ do
112+ sem <- mkSem
113+ withServer sem $ \receivedMessages -> do
114+ withScribe (\cfg -> cfg { logzIOScribeConfiguration_bufferItems = 100, logzIOScribeConfiguration_bufferTimeout = 100}) $ \scribe -> do
115+ (liPush scribe) (genericLog "message 1")
116+ (liPush scribe) (genericLog "message 2")
117+ messages <- atomically (flushTQueue receivedMessages)
118+ let allLogs = receivedLog_message <$> mconcat (requestLog_logs <$> messages)
119+ allLogs @?= ["message 1", "message 2"]
120+ ]
121+ where
122+ second = 1000000
123+ timeoutFailure waitTime f = do
124+ res <- timeout waitTime f
125+ case res of
126+ Nothing -> assertFailure ("Operation timed out after " <> show waitTime <> "us")
127+ Just x -> pure x
128+ withLock = withResource (atomically (newTSem 1)) (const (return ()))
129+ withServer sem f = bracket_ (atomically (waitTSem sem)) (atomically (signalTSem sem)) $ do
130+ receivedMessages <- atomically newTQueue
131+ withAsync (startServer receivedMessages) $ \_async -> f receivedMessages
132+ testAPIToken = "bigSECRET"
133+ genericLog message = Item
134+ { _itemApp = "katip-logzio"
135+ , _itemEnv = "test"
136+ , _itemSeverity = InfoS
137+ , _itemThread = ThreadIdText "42"
138+ , _itemHost = "localhost"
139+ , _itemProcess = 111
140+ , _itemPayload = ()
141+ , _itemMessage = message
142+ , _itemTime = mkUTCTime 2019 1 2 3 4 5
143+ , _itemNamespace = "test"
144+ , _itemLoc = Nothing
145+ }
146+ startServer receivedMessages = Scotty.scottyOpts serverOpts $ do
147+ Scotty.post "/" $ do
148+ token <- APIToken <$> Scotty.param "token"
149+ if token == testAPIToken
150+ then do
151+ bodyLines <- LBS8.lines <$> Scotty.body
152+ receivedLogs <- mapM (either error pure . A.eitherDecode) bodyLines
153+ let requestLog = RequestLog
154+ { requestLog_token = token
155+ , requestLog_logs = receivedLogs
156+ }
157+ liftIO (atomically (writeTQueue receivedMessages requestLog))
158+ else Scotty.status HTypes.unauthorized401
159+ serverOpts = Scotty.Options
160+ { Scotty.verbose = 0
161+ , Scotty.settings = Warp.setPort 1337 Warp.defaultSettings
162+ }
163+ withScribe modConfig =
164+ bracket (mkLogzIOScribe (modConfig scribeConfig) (const (pure True)) V3) scribeFinalizer
165+ scribeConfig = LogzIOScribeConfiguration
166+ { logzIOScribeConfiguration_bufferItems = 5
167+ , logzIOScribeConfiguration_bufferTimeout = 10
168+ , logzIOScribeConfiguration_scheme = HTTP
169+ , logzIOScribeConfiguration_host = Host "127.0.0.1"
170+ , logzIOScribeConfiguration_port = Port 1337
171+ , logzIOScribeConfiguration_token = testAPIToken
172+ , logzIOScribeConfiguration_retry = defaultRetryPolicy
173+ , logzIOScribeConfiguration_onError = const (pure ())
174+ }
175+
176+
177+
178+data RequestLog = RequestLog
179+ { requestLog_token :: APIToken
180+ , requestLog_logs :: [ReceivedLog]
181+ }
182+
183+
184+-- | Simplified log that only parses the fields we want to assert on
185+data ReceivedLog = ReceivedLog
186+ { receivedLog_message :: Text
187+ }
188+
189+
190+instance FromJSON ReceivedLog where
191+ parseJSON = withObject "ReceivedLog" $ \o -> do
192+ message <- o .: "message"
193+ pure $ ReceivedLog
194+ { receivedLog_message = message
195+ }
196+
197+
198+-------------------------------------------------------------------------------
199+bufferItemTests :: TestTree
200+bufferItemTests = testGroup "bufferItem"
201+ [ testProperty "0 or negative maximum items always flushes immediately" $ property $ do
202+ item <- forAllWith showSimplePayloadItem genSimplePayloadItem
203+ verbosity <- forAll genVerbosity
204+ maxItems <- forAll (Gen.int (Range.linear minBound 0))
205+ bulkBuffer <- forAllWith inspectBulkBuffer genBulkBuffer
206+ let logAction = bufferItem' reducedMaxPayloadBytes reducedMaxLogLineLength maxItems verbosity item bulkBuffer
207+ assertFlushNow logAction
208+ , testProperty "never buffers too many items" $ property $ do
209+ item <- forAllWith showSimplePayloadItem genSimplePayloadItem
210+ verbosity <- forAll genVerbosity
211+ maxItems <- forAll (Gen.int (Range.linear 0 100))
212+ bulkBuffer <- forAllWith inspectBulkBuffer genBulkBuffer
213+ let logAction = bufferItem' reducedMaxPayloadBytes reducedMaxLogLineLength maxItems verbosity item bulkBuffer
214+ case logAction of
215+ Buffered newBuffer -> HH.assert (bulkBuffer_itemCount newBuffer <= maxItems)
216+ _ -> pure ()
217+ , testProperty "never produces too many bytes" $ property $ do
218+ item <- forAllWith showSimplePayloadItem genSimplePayloadItem
219+ verbosity <- forAll genVerbosity
220+ maxItems <- forAll (Gen.int (Range.linear 0 100))
221+ bulkBuffer <- forAllWith inspectBulkBuffer genBulkBuffer
222+ let logAction = bufferItem' reducedMaxPayloadBytes reducedMaxLogLineLength maxItems verbosity item bulkBuffer
223+ case logAction of
224+ Buffered newBuffer -> HH.assert (bulkBuffer_bytesUsed newBuffer <= reducedMaxPayloadBytes)
225+ _ -> pure ()
226+ , testProperty "never flushes more than 1 item" $ property $ do
227+ item <- forAllWith showSimplePayloadItem genSimplePayloadItem
228+ verbosity <- forAll genVerbosity
229+ maxItems <- forAll (Gen.int (Range.linear minBound 0))
230+ bulkBuffer <- forAllWith inspectBulkBuffer genBulkBuffer
231+ let logAction = bufferItem' reducedMaxPayloadBytes reducedMaxLogLineLength maxItems verbosity item bulkBuffer
232+ case logAction of
233+ FlushNow _ newBuffer -> bulkBuffer_itemCount newBuffer === 1
234+ _ -> pure ()
235+ , testProperty "flushes when the buffer is exactly at the limit" $ property $ do
236+ maxItems <- forAll (Gen.int (Range.linear 1 20))
237+ item <- forAllWith showSimplePayloadItem genSimplePayloadItem
238+ bulkBuffer <- forAllWith inspectBulkBuffer ((\b -> b { bulkBuffer_itemCount = maxItems - 1 }) <$> genBulkBuffer)
239+ verbosity <- forAll genVerbosity
240+ let logAction = bufferItem' maxBound maxBound maxItems verbosity item bulkBuffer
241+ case logAction of
242+ FlushNow _ _ -> pure ()
243+ Buffered _ -> fail "Expected a FlushNow but got a Buffered"
244+ ]
245+ where
246+ assertFlushNow (FlushNow _ _) = pure ()
247+ assertFlushNow (Buffered _) = fail ("Expceted FlushNow but got a Buffered")
248+ inspectBulkBuffer (BulkBuffer used _ itemCount) =
249+ "BulkBuffer (" <> show used <> ") ... " <> show itemCount
250+
251+
252+-------------------------------------------------------------------------------
253+-- | Reduced max log line length for more space and time efficient tests
254+reducedMaxLogLineLength :: Bytes
255+reducedMaxLogLineLength = Bytes (bytes maxLogLineLength `div` limitScalingFactor)
256+
257+
258+reducedMaxPayloadBytes :: Bytes
259+reducedMaxPayloadBytes = Bytes (bytes maxPayloadBytes `div` limitScalingFactor)
260+
261+
262+limitScalingFactor :: Int64
263+limitScalingFactor = 10
264+
265+-------------------------------------------------------------------------------
266+showSimplePayloadItem :: Item SimpleLogPayload -> String
267+showSimplePayloadItem item =
268+ show (item { _itemPayload = [(k, toJSON v) | (k, AnyLogPayload v) <- unSimpleLogPayload (_itemPayload item)]})
269+
270+
271+-------------------------------------------------------------------------------
272+renderLineTruncatedTests :: TestTree
273+renderLineTruncatedTests = testGroup "renderLineTruncated"
274+ [ testProperty "always produces an accurate byte count" $ property $ do
275+ item <- forAllWith showSimplePayloadItem genSimplePayloadItem
276+ verbosity <- forAll genVerbosity
277+ let (builder, Bytes byteCount) = renderLineTruncated' reducedMaxLogLineLength verbosity item
278+ builderLength builder === byteCount
279+ , testProperty "always produces a payload smaller than the line limit" $ property $ do
280+ let tinyMaxLineLength = 1024 -- set a really low one for speed
281+ let lowerBound = fromIntegral (bytes tinyMaxLineLength - 100)
282+ let upperBound = fromIntegral (bytes tinyMaxLineLength + 100)
283+ bigMessage <- forAll (genLogStr' (Range.linear lowerBound upperBound))
284+ item <- (\item -> item { _itemMessage = bigMessage}) <$> forAllWith showSimplePayloadItem genSimplePayloadItem
285+ verbosity <- forAll genVerbosity
286+ let (_, byteCount) = renderLineTruncated' tinyMaxLineLength verbosity item
287+ HH.assert (byteCount <= reducedMaxLogLineLength)
288+ ]
289+
290+
291+-------------------------------------------------------------------------------
292+builderLength :: BB.Builder -> Int64
293+builderLength = LBS.length . BB.toLazyByteString
294+
295+-------------------------------------------------------------------------------
296+genVerbosity :: Gen Verbosity
297+genVerbosity = Gen.enumBounded
298+
299+
300+-------------------------------------------------------------------------------
301+genBulkBuffer :: Gen BulkBuffer
302+genBulkBuffer = do
303+ bytesUsed <- genBytes
304+ -- generating a large bytestring string with hedghog seems quite slow, instead we'll generate a smaller string and replicate and merge it
305+ payloadMultiplier <- Gen.int (Range.linear 1 100)
306+ payloadSegment <- genBuilder (Range.linear 0 (round (toRational (bytes reducedMaxPayloadBytes) * 0.05)))
307+ let payload = mconcat (replicate payloadMultiplier payloadSegment)
308+ itemCount <- Gen.int (Range.linear 0 maxBound)
309+ pure $ BulkBuffer
310+ { bulkBuffer_bytesUsed = bytesUsed
311+ , bulkBuffer_payload = payload
312+ , bulkBuffer_itemCount = itemCount
313+ }
314+
315+
316+-------------------------------------------------------------------------------
317+genBytes :: Gen Bytes
318+genBytes = Bytes <$> Gen.int64 (Range.linear 0 maxBound)
319+
320+
321+-------------------------------------------------------------------------------
322+genBuilder :: Range Int -> Gen BB.Builder
323+genBuilder size = BB.byteString <$> Gen.bytes size
324+
325+
326+-------------------------------------------------------------------------------
327+genSimplePayloadItem :: Gen (Item SimpleLogPayload)
328+genSimplePayloadItem = do
329+ app <- genNamespace
330+ env <- genEnvironment
331+ severity <- genSeverity
332+ thread <- genThreadIdText
333+ host <- genHostName
334+ process <- genProcessID
335+ payload <- genSimpleLogPayload
336+ message <- genLogStr
337+ time <- genUTCTime
338+ namespace <- genNamespace
339+ loc <- Gen.maybe genLoc
340+ pure $ Item
341+ { _itemApp = app
342+ , _itemEnv = env
343+ , _itemSeverity = severity
344+ , _itemThread = thread
345+ , _itemHost = host
346+ , _itemProcess = process
347+ , _itemPayload = payload
348+ , _itemMessage = message
349+ , _itemTime = time
350+ , _itemNamespace = namespace
351+ , _itemLoc = loc
352+ }
353+
354+
355+genNamespace :: Gen Namespace
356+genNamespace = Namespace <$> Gen.list (Range.linear 0 20) genSmallText
357+
358+
359+genEnvironment :: Gen Environment
360+genEnvironment = Environment <$> genSmallText
361+
362+
363+genSeverity :: Gen Severity
364+genSeverity = Gen.enumBounded
365+
366+
367+genThreadIdText :: Gen ThreadIdText
368+genThreadIdText = ThreadIdText <$> genSmallText
369+
370+
371+genHostName :: Gen HostName
372+genHostName = genSmallString
373+
374+
375+genProcessID :: Gen ProcessID
376+genProcessID = Gen.enumBounded
377+
378+
379+genLogStr :: Gen LogStr
380+genLogStr = genLogStr' (Range.linear 0 1024)
381+
382+
383+genLogStr' :: Range Int -> Gen LogStr
384+genLogStr' sizeRange = ls <$> Gen.text sizeRange Gen.unicode
385+
386+
387+genUTCTime :: Gen UTCTime
388+genUTCTime = posixSecondsToUTCTime . fromIntegral
389+ <$> Gen.int (Range.linear 0 maxBound)
390+
391+
392+genLoc :: Gen Loc
393+genLoc = Loc
394+ <$> genSmallString
395+ <*> genSmallString
396+ <*> genSmallString
397+ <*> genCharPos
398+ <*> genCharPos
399+ where
400+ genCharPos = (,)
401+ <$> Gen.int (Range.linear 0 maxBound)
402+ <*> Gen.int (Range.linear 0 maxBound)
403+
404+
405+genSimpleLogPayload :: Gen SimpleLogPayload
406+genSimpleLogPayload = SimpleLogPayload
407+ <$> Gen.list (Range.linear 0 20) genPair
408+ where
409+ genPair = (,)
410+ <$> genSmallText
411+ <*> genAnyLogPayload
412+
413+
414+genAnyLogPayload :: Gen AnyLogPayload
415+genAnyLogPayload = AnyLogPayload
416+ <$> genValue
417+
418+
419+genValue :: Gen Value
420+genValue = Gen.recursive Gen.choice nonRecursiveGens recursiveGens
421+ where
422+ nonRecursiveGens =
423+ [ pure A.Null
424+ , A.String <$> genSmallText
425+ , A.Number <$> genScientific
426+ , A.Bool <$> Gen.bool
427+ ]
428+ recursiveGens =
429+ [ A.Object <$> genObject
430+ , A.Array <$> genArray
431+ ]
432+ genObject = HM.fromList <$> Gen.list (Range.linear 0 20) genPair
433+ where
434+ genPair = (,) <$> genSmallText <*> genValue
435+ genArray = V.fromList <$>
436+ Gen.list (Range.linear 0 20) genValue
437+
438+
439+
440+genScientific :: Gen Scientific
441+genScientific = scientific
442+ <$> Gen.integral (Range.linear (-999999) 999999)
443+ <*> Gen.int (Range.linear (-999) (999))
444+
445+
446+genSmallText :: Gen Text
447+genSmallText = Gen.text (Range.linear 0 128) Gen.unicode
448+
449+
450+genSmallString :: Gen String
451+genSmallString = Gen.string (Range.linear 0 128) Gen.unicode
452+
453+
454+-------------------------------------------------------------------------------
455+mkUTCTime :: Integer -> Int -> Int -> DiffTime -> DiffTime -> DiffTime -> UTCTime
456+mkUTCTime y m d hr mn s = UTCTime day dt
457+ where
458+ day = mkDay y m d
459+ dt = s + 60 * mn + 60 * 60 * hr
460+
461+
462+-------------------------------------------------------------------------------
463+mkDay :: Integer -> Int -> Int -> Day
464+mkDay y m d = day
465+ where
466+ Just day = fromGregorianValid y m d
katip-logzio/test/Main.hsnew file+22
@@ -0,0 +1,22 @@
1+module Main
2+ ( main
3+ ) where
4+
5+
6+-------------------------------------------------------------------------------
7+import Test.Tasty
8+-------------------------------------------------------------------------------
9+import qualified Katip.Tests.Scribes.LogzIO.HTTPS
10+-------------------------------------------------------------------------------
11+
12+
13+
14+main :: IO ()
15+main = defaultMain tests
16+
17+
18+-------------------------------------------------------------------------------
19+tests :: TestTree
20+tests = testGroup "katip-logzio"
21+ [ Katip.Tests.Scribes.LogzIO.HTTPS.tests
22+ ]
katip/README.md+4−1
@@ -16,7 +16,7 @@ Kâtip (pronounced kah-tip) is the Turkish word for scribe.
1616 add context that will be automatically merged in with existing log
1717 context.
1818
19−* *Easy to Integration:* Katip was designed to be easily integrated
19+* *Easy Integration:* Katip was designed to be easily integrated
2020 into existing monads. By using typeclasses for logging facilities,
2121 individual subsystems and even libraries can easily add their own
2222 namespacing and context without having any knowledge of their
@@ -84,3 +84,6 @@ stack.
8484* [Leonid Onokhov](https://github.com/sopvop)
8585* [Alexander Vershilov](https://github.com/qnikst)
8686* [Chris Martin](https://github.com/chris-martin)
87+* [Domen Kožar](https://github.com/domenkozar)
88+* [Tristan Bull](https://github.com/tmbull)
89+* [Aleksey Khudyakov](https://github.com/Shimuuar)
katip/bench/Main.hs+5−2
@@ -92,7 +92,7 @@ setup dest = do
9292 return (tmp </> "katip-bench.log")
9393 DevNull -> return ("/dev/null")
9494 h <- openFile outFile WriteMode
95− s <- mkHandleScribe ColorIfTerminal h DebugS V0
95+ s <- mkHandleScribe ColorIfTerminal h (permitItem DebugS) V0
9696 let cleanupHandle = hClose h `finally` (when (dest == TempFile) (removeLink outFile))
9797 return s { scribeFinalizer = scribeFinalizer s `finally` cleanupHandle}
9898
@@ -101,4 +101,7 @@ setup dest = do
101101deriving instance NFData ThreadIdText
102102
103103instance NFData Scribe where
104− rnf (Scribe a b) = (a :: Item ExPayload -> IO ()) `seq` b `seq` ()
104+ rnf (Scribe a b p) = (a :: Item ExPayload -> IO ())
105+ `seq` b
106+ `seq` (p :: Item ExPayload -> IO Bool)
107+ `seq` ()
katip/changelog.md+114
@@ -1,3 +1,117 @@
1+0.8.5.0
2+=======
3+* Add MonadResource instances [#121](https://github.com/Soostone/katip/pull/121)
4+
5+0.8.4.0
6+=======
7+* Add support for unliftio-core 0.2 while maintaining backwards-compatible support for 0.1
8+
9+0.8.3.0
10+=======
11+
12+* Add `logKatipItem` function and reimplement `logItem` to use it. [vlatkoB](https://github.com/vlatkoB)
13+
14+0.8.2.0
15+=======
16+* Add `MonadFail` instances for `base` >= 4.9.0.0 [Koray Al](https://github.com/korayal)
17+
18+0.8.1.0
19+=======
20+* Export `logLoc`. Credit to [Brian McKenna](https://github.com/puffnfresh)
21+
22+0.8.0.0
23+=======
24+* **Breaking change**: switch from severity based log level filtering to
25+ user-definable `PermitFunc`. To upgrade, any time you initialize one
26+ of the standard scribes, rather than providing a severity like
27+ `InfoS`, instead pass in `(permitItem InfoS)`. Credit to
28+ [vlatkoB](https://github.com/vlatkoB).
29+* Update some misformatted and outdated docs.
30+* Set lower bound of base to 4.9.0.0, the version shipped with GHC
31+ 8.0.2. Support for GHC versions older than that is removed.
32+* Update default formatter of Handle scribe to format pid as `[PID
33+ 1234]` instead of `[1234]` and `[ThreadId 1234]` instead of
34+ `[1234]`.
35+* Add missing `Bounded` instance for `Verbosity`. This allows you to
36+ easily enumerate all verbosities via `[minBound..maxBound]`.
37+
38+0.7.0.0
39+=======
40+* Exclusively use HasCallstack for location reporting in errors. Credit to [Domen Kožar](https://github.com/domenkozar)
41+* Drop support for GHC 7.x
42+
43+0.6.3.0
44+=======
45+* Add `ToJSON`/`FromJSON` instances for Verbosity. Credit to [Aleksey Khudyakov](https://github.com/Shimuuar).
46+
47+0.6.2.1
48+=======
49+* Add `mkHandleScribeWithFormatter`. This allows control over the format of log items going to a handle. Credit to [Tristan Bull](https://github.com/tmbull) for the implementation.
50+* Documentation fixes. Credit to [Domen Kožar](https://github.com/domenkozar) for the implementation.
51+* Add `jsonFormat` handle formatter.
52+* Deprecate `formatItem` which is now replaced by `bracketFormat`.
53+
54+0.6.1.0
55+=======
56+* Loosen deps
57+* Support latest STM
58+
59+0.6.0.0
60+=======
61+* Drop ListT instance due to deprecations.
62+* Loosen dependencies.
63+
64+0.5.5.1
65+=======
66+* Decrease default timer resolution from 1s to 1ms.
67+
68+0.5.5.0
69+=======
70+* Export `runNoLoggingT` and `NoLoggingT` constructor.
71+* Delegate MonadReader instance for `NoLoggingT`.
72+
73+0.5.4.0
74+=======
75+* Loosen bounds on resourcet and template-haskell.
76+* Add convenience function askLoggerIO.
77+
78+0.5.3.0
79+=======
80+* Add MonadUnliftIO instances.
81+* Add NoLoggingT
82+
83+0.5.2.0
84+=======
85+* Allow newer versions of either by conditionally adding instances for the removed EitherT interface.
86+
87+0.5.1.0
88+=======
89+* Add mkFileScribe, a specialization of mkHandleScribe for files that manages the handle automatically.
90+
91+0.5.0.4
92+=======
93+* Loosen Win32 upper bound to run with GHC 8.2 on Windows.
94+
95+0.5.0.3
96+=======
97+* Add worked example of Katip/KatipContext to the haddocks.
98+
99+0.5.0.2
100+=======
101+* Export Katip.Compat for Windows users.
102+
103+0.5.0.1
104+=======
105+* Fix numeric formatting in Handle scribe.
106+* Bump deps for GHC 8.2.1
107+
108+0.5.0.0
109+=======
110+* Improved documentation.
111+* Add built-in buffering to scribes.
112+ Scribes now allocate a bounded queue (with configurable size). Rather than writes being synchronous to all scribes, they simply attempt to write into the bounded queue of each scribe. If any of the scribes is too far behind and the queue is full, the write is dropped. This also means that closing scribes is now an IO operation that happens synchrounsly.
113+* Added local-like functions to Katip and KatipContext typeclasses. This allows us to generalize `katipNoLogging`, `katipAddNamespace`, and `katipAddContext` to be available to anything with a `Katip` or `KatipContext` instance rather than having to reimplement these functions all the time.
114+
11150.4.1.0
2116=======
3117* Add Katip instances for Strict StateT, WriterT, RWST.
katip/examples/example.hs+24−22
@@ -13,6 +13,7 @@ module Main
1313
1414-------------------------------------------------------------------------------
1515import qualified Control.Applicative as A
16+import Control.Exception
1617import Control.Monad.Base
1718import Control.Monad.Reader
1819import Control.Monad.Trans.Control
@@ -32,28 +33,29 @@ main = do
3233 -- fields permitted for Verbosity 2 and will throw out Debug
3334 -- messages entirely. Note that katip provides facilities like
3435 -- 'unregisterScribe' and 'registerScribe' to make it possible to
35− -- hot-swap scribes at runtime if you need to.
36− handleScribe <- mkHandleScribe ColorIfTerminal stdout InfoS V2
37− le <- registerScribe "stdout" handleScribe defaultScribeSettings =<< initLogEnv "MyApp" "production"
38− let s = MyState M.mempty mempty le
39− runStack s $ do
40− $(logTM) InfoS "Started"
41− -- this will add "confrabulation" to the current namespace, making
42− -- logs made under this block have the namespace of
43− -- "main.confrabulation". Further, ConfrabLogCTX's key/value
44− -- context will also get merged with the context above it. You can
45− -- use this to stack up various contextual details throughout your
46− -- code and they will be flattened out and combined in the log
47− -- output.
48− katipAddNamespace "confrabulation" $ katipAddContext (ConfrabLogCTX 42) $ do
49− $(logTM) DebugS "Confrabulating widgets, with extra namespace and context"
50− confrabulateWidgets
51− $(logTM) InfoS "Namespace and context are back to normal"
52− katipNoLogging $
53− $(logTM) DebugS "You'll never see this log message!"
54− -- This blocking call flushes all in-flight messages through
55− -- handlers and then finalizes each scribe.
56− void (closeScribes le)
36+ -- hot-swap scribes at runtime if you need to. 'closeScribes' is
37+ -- blocking and flushes all messages out of a scribe and cleans up
38+ -- resources that were allocated at creation.
39+ handleScribe <- mkHandleScribe ColorIfTerminal stdout (permitItem InfoS) V2
40+ let mkLogEnv = registerScribe "stdout" handleScribe defaultScribeSettings =<< initLogEnv "MyApp" "production"
41+ bracket mkLogEnv closeScribes $ \le -> do
42+ let s = MyState M.mempty mempty le
43+ runStack s $ do
44+ $(logTM) InfoS "Started"
45+ -- this will add "confrabulation" to the current namespace, making
46+ -- logs made under this block have the namespace of
47+ -- "main.confrabulation". Further, ConfrabLogCTX's key/value
48+ -- context will also get merged with the context above it. You can
49+ -- use this to stack up various contextual details throughout your
50+ -- code and they will be flattened out and combined in the log
51+ -- output.
52+ katipAddNamespace "confrabulation" $ katipAddContext (ConfrabLogCTX 42) $ do
53+ $(logTM) DebugS "Confrabulating widgets, with extra namespace and context"
54+ confrabulateWidgets
55+ $(logTM) InfoS "Namespace and context are back to normal"
56+ katipNoLogging $
57+ $(logTM) DebugS "You'll never see this log message!"
58+
5759
5860-------------------------------------------------------------------------------
5961newtype ConfrabLogCTX = ConfrabLogCTX Int
katip/examples/example_lens.hs+23−22
@@ -13,6 +13,7 @@ module Main
1313
1414-------------------------------------------------------------------------------
1515import Control.Applicative as A
16+import Control.Exception
1617import Control.Lens hiding ((.=))
1718import Control.Monad.Base
1819import Control.Monad.Reader
@@ -47,28 +48,28 @@ main = do
4748 -- fields permitted for Verbosity 2 and will throw out Debug
4849 -- messages entirely. Note that katip provides facilities like
4950 -- 'unregisterScribe' and 'registerScribe' to make it possible to
50− -- hot-swap scribes at runtime if you need to.
51− handleScribe <- mkHandleScribe ColorIfTerminal stdout InfoS V2
52− le <- registerScribe "stdout" handleScribe defaultScribeSettings =<< initLogEnv "MyApp" "production"
53− let s = MyState M.mempty mempty le
54− runStack s $ do
55− $(logTM) InfoS "Started"
56− -- this will add "confrabulation" to the current namespace, making
57− -- logs made under this block have the namespace of
58− -- "main.confrabulation". Further, ConfrabLogCTX's key/value
59− -- context will also get merged with the context above it. You can
60− -- use this to stack up various contextual details throughout your
61− -- code and they will be flattened out and combined in the log
62− -- output.
63− katipAddNamespace "confrabulation" $ katipAddContext (ConfrabLogCTX 42) $ do
64− $(logTM) DebugS "Confrabulating widgets, with extra namespace and context"
65− confrabulateWidgets
66− $(logTM) InfoS "Namespace and context are back to normal"
67− katipNoLogging $
68− $(logTM) DebugS "You'll never see this log message!"
69− -- This blocking call flushes all in-flight messages through
70− -- handlers and then finalizes each scribe.
71− void (closeScribes le)
51+ -- hot-swap scribes at runtime if you need to. 'closeScribes' is
52+ -- blocking and flushes all messages out of a scribe and cleans up
53+ -- resources that were allocated at creation.
54+ handleScribe <- mkHandleScribe ColorIfTerminal stdout (permitItem InfoS) V2
55+ let mkLogEnv = registerScribe "stdout" handleScribe defaultScribeSettings =<< initLogEnv "MyApp" "production"
56+ bracket mkLogEnv closeScribes $ \le -> do
57+ let s = MyState M.mempty mempty le
58+ runStack s $ do
59+ $(logTM) InfoS "Started"
60+ -- this will add "confrabulation" to the current namespace, making
61+ -- logs made under this block have the namespace of
62+ -- "main.confrabulation". Further, ConfrabLogCTX's key/value
63+ -- context will also get merged with the context above it. You can
64+ -- use this to stack up various contextual details throughout your
65+ -- code and they will be flattened out and combined in the log
66+ -- output.
67+ katipAddNamespace "confrabulation" $ katipAddContext (ConfrabLogCTX 42) $ do
68+ $(logTM) DebugS "Confrabulating widgets, with extra namespace and context"
69+ confrabulateWidgets
70+ $(logTM) InfoS "Namespace and context are back to normal"
71+ katipNoLogging $
72+ $(logTM) DebugS "You'll never see this log message!"
7273
7374
7475-------------------------------------------------------------------------------
katip/katip.cabal+28−26
@@ -1,5 +1,5 @@
11name: katip
2−version: 0.4.1.0
2+version: 0.8.5.0
33synopsis: A structured logging framework.
44description:
55 Katip is a structured logging framework. See README.md for more details.
@@ -8,7 +8,7 @@ license: BSD3
88license-file: LICENSE
99author: Ozgun Ataman, Michael Xavier
1010maintainer: michael.xavier@soostone.com
11−copyright: Soostone Inc, 2015-2016
11+copyright: Soostone Inc, 2015-2017
1212category: Data, Text, Logging
1313homepage: https://github.com/Soostone/katip
1414bug-reports: https://github.com/Soostone/katip/issues
@@ -25,7 +25,7 @@ extra-source-files:
2525 test/Katip/Tests/Scribes/Handle.hs
2626 test/Katip/Tests/Scribes/Handle-text.golden
2727 test/Katip/Tests/Format/Time.hs
28−tested-with: GHC == 7.8.4, GHC== 7.10.3
28+tested-with: GHC == 8.0.1, GHC == 8.2.2, GHC == 8.4.3
2929
3030source-repository head
3131 type: git
@@ -53,30 +53,32 @@ library
5353 TemplateHaskell
5454 OverloadedStrings
5555
56− build-depends: base >=4.5 && <5
57− , aeson >=0.6 && <1.2
56+ build-depends: base >=4.9.0.0 && <5
57+ , aeson >=0.6
5858 , async < 3.0.0.0
59− , auto-update >= 0.1 && < 0.2
60− , bytestring >= 0.9 && < 0.11
61− , containers >=0.4 && <0.6
62− , either >= 4 && < 4.5
59+ , auto-update >= 0.1
60+ , bytestring >= 0.9
61+ , containers >=0.4
62+ , either >= 4
6363 , safe-exceptions >= 0.1.0.0
64− , hostname >=1.0 && <1.1
65− , old-locale >= 1.0 && < 1.1
66− , string-conv >= 0.1 && < 0.2
67− , template-haskell >= 2.8 && < 2.12
68− , text >= 0.11 && <1.3
69− , time >= 1 && < 1.7
70− , transformers >= 0.3 && < 0.6
64+ , hostname >=1.0
65+ , old-locale >= 1.0
66+ , string-conv >= 0.1
67+ , template-haskell >= 2.8
68+ , text >= 0.11
69+ , time >= 1
70+ , transformers >= 0.3
7171 , transformers-compat
72− , unordered-containers >= 0.2 && < 0.3
73− , monad-control >= 1.0 && < 1.1
74− , mtl >= 2.0 && < 2.3
75− , transformers-base >= 0.3 && < 0.6
76− , resourcet >= 1.1 && < 1.2
77− , microlens >= 0.2.0.0 && < 0.5
78− , microlens-th >= 0.1.0.0 && < 0.5
72+ , unordered-containers >= 0.2
73+ , monad-control >= 1.0
74+ , mtl >= 2.0
75+ , transformers-base >= 0.3
76+ , resourcet >= 1.1
77+ , scientific >= 0.3.3.0
78+ , microlens >= 0.2.0.0
79+ , microlens-th >= 0.1.0.0
7980 , semigroups
81+ , unliftio-core >= 0.1
8082 , stm >= 2.4
8183
8284 hs-source-dirs: src
@@ -85,8 +87,8 @@ library
8587 if flag(lib-Werror)
8688 ghc-options: -Werror
8789 if os(windows)
88− build-depends: Win32 >=2.3 && <2.5
89− other-modules: Katip.Compat
90+ build-depends: Win32 >=2.3 && <2.9
91+ exposed-modules: Katip.Compat
9092 else
9193 build-depends: unix >= 2.5 && <2.8
9294
@@ -112,7 +114,7 @@ test-suite test
112114 , template-haskell
113115 , text
114116 , time
115− , time-locale-compat >= 0.1.0.1 && < 0.2
117+ , time-locale-compat >= 0.1.0.1
116118 , directory
117119 , regex-tdfa
118120 , unordered-containers
katip/src/Katip.hs+92−9
@@ -1,7 +1,78 @@
1−-- | Includes all of the API's youll need to use Katip. Be sure to
1+-- | Includes all of the APIs youll need to use Katip. Be sure to
22-- check out the included @examples@ directory for an example of
33-- usage.
44--
5+-- Here's a basic example:
6+--
7+-- @
8+--
9+-- import Control.Exception
10+-- import Katip
11+--
12+-- main :: IO ()
13+-- main = do
14+-- handleScribe <- mkHandleScribe ColorIfTerminal stdout (permitItem InfoS) V2
15+-- let makeLogEnv = registerScribe "stdout" handleScribe defaultScribeSettings =<< initLogEnv \"MyApp\" \"production\"
16+-- -- closeScribes will stop accepting new logs, flush existing ones and clean up resources
17+-- bracket makeLogEnv closeScribes $ \le -> do
18+-- let initialContext = () -- this context will be attached to every log in your app and merged w/ subsequent contexts
19+-- let initialNamespace = "main"
20+-- runKatipContextT le initialContext initialNamespace $ do
21+-- $(logTM) InfoS "Hello Katip"
22+-- -- This adds a namespace to the current namespace and merges a piece of contextual data into your context
23+-- katipAddNamespace "additional_namespace" $ katipAddContext (sl "some_context" True) $ do
24+-- $(logTM) WarningS "Now we're getting fancy"
25+-- katipNoLogging $ do
26+-- $(logTM) DebugS "You will never see this!"
27+--
28+-- @
29+--
30+-- Another common case that you have some sort of App monad that's
31+-- based on ReaderT with some Config state. This is a perfect place to
32+-- insert read-only katip state:
33+--
34+-- @
35+--
36+-- import Katip as K
37+--
38+-- data Config = Config {
39+-- logNamespace :: K.Namespace
40+-- , logContext :: K.LogContexts
41+-- , logEnv :: K.LogEnv
42+-- -- whatever other read-only config you need
43+-- }
44+--
45+-- newtype App m a = App {
46+-- unApp :: ReaderT Config m a
47+-- } deriving (Functor, Applicative, Monad, MonadIO, MonadReader Config) -- these are necessary
48+--
49+--
50+-- -- These instances get even easier with lenses!
51+-- instance (MonadIO m) => K.Katip (App m) where
52+-- getLogEnv = asks logEnv
53+-- -- with lens:
54+-- -- getLogEnv = view logEnv
55+-- localLogEnv f (App m) = App (local (\\s -> s { logEnv = f (logEnv s)}) m)
56+-- -- with lens:
57+-- -- localLogEnv f (App m) = App (local (over logEnv f) m)
58+--
59+--
60+-- instance (MonadIO m) => K.KatipContext (App m) where
61+-- getKatipContext = asks logContext
62+-- -- with lens:
63+-- -- getKatipContext = view logContext
64+-- localKatipContext f (App m) = App (local (\\s -> s { logContext = f (logContext s)}) m)
65+-- -- with lens:
66+-- -- localKatipContext f (App m) = App (local (over logContext f) m)
67+-- getKatipNamespace = asks logNamespace
68+-- -- with lens:
69+-- -- getKatipNamespace = view logNamespace
70+-- localKatipNamespace f (App m) = App (local (\\s -> s { logNamespace = f (logNamespace s)}) m)
71+-- -- with lens:
72+-- -- localKatipNamespace f (App m) = App (local (over logNamespace f) m)
73+--
74+-- @
75+--
576-- To get up and running, the workflow is generally:
677--
778-- * Set up a 'LogEnv' using 'initLogEnv'.
@@ -40,7 +111,7 @@ module Katip
40111 , Scribe (..)
41112 , LogEnv (..)
42113 , SimpleLogPayload, sl
43− , defaultScribeSettings --TODO: lenses and whatnot to avoid breakage
114+ , defaultScribeSettings
44115 , ScribeSettings
45116 , scribeBufferSize
46117 , _scribeBufferSize
@@ -64,7 +135,7 @@ module Katip
64135 , logEnvTimer
65136 , logEnvScribes
66137
67− -- * A Built-in Monad For Logging
138+ -- * A Built-in Monad For Simple Logging
68139 , KatipT (..)
69140 , runKatipT
70141
@@ -82,29 +153,44 @@ module Katip
82153 , LogStr (..)
83154 , logStr, ls, showLS
84155
85− -- ** 'Katip' LoggingFunctions
156+ -- ** 'Katip' Logging Functions
86157 -- $katiplogging
87158 , Katip (..)
88159 , logF
89160 , logMsg
90161 , logT
162+ , logLoc
91163 , logItem
164+ , logKatipItem
92165 , logException
93− -- ** 'KatipContext' Logging Functions
166+ -- ** 'KatipContext': Logging With Context
94167 -- $katipcontextlogging
95168 , KatipContext (..)
96169 , logFM
97170 , logTM
171+ , logLocM
98172 , logItemM
99173 , logExceptionM
100174 , AnyLogContext
101175 , LogContexts, liftPayload
176+ -- *** Temporarily Changing Logging Behavior
177+ , katipAddNamespace
178+ , katipAddContext
179+ , katipNoLogging
102180
103181 -- * Included Scribes
104182 , mkHandleScribe
183+ , mkHandleScribeWithFormatter
184+ , mkFileScribe
105185 , ColorStrategy (..)
186+ , ItemFormatter
187+ , bracketFormat
188+ , jsonFormat
106189
107190 -- * Tools for implementing Scribes
191+ , PermitFunc
192+ , permitAND
193+ , permitOR
108194 , permitItem
109195 , payloadObject
110196 , itemJson
@@ -112,9 +198,6 @@ module Katip
112198 -- * KatipContextT - Utility transformer that provides Katip and KatipContext instances
113199 , KatipContextT
114200 , runKatipContextT
115− , katipAddNamespace
116− , katipAddContext
117− , katipNoLogging
118201 ) where
119202
120203-------------------------------------------------------------------------------
@@ -144,7 +227,7 @@ doDatabaseThings = do
144227@
145228
146229Another pain point to look out for is nesting actions that log in
147−eachother. Let's say you were writing a web app. You want to capture
230+each other. Let's say you were writing a web app. You want to capture
148231some detail such as the user's ID in the logs, but you also want that
149232info to show up in doDatabaseThings' logs so you can associate those
150233two pieces of information:
katip/src/Katip/Core.hs+248−110
@@ -7,7 +7,6 @@
77{-# LANGUAGE FlexibleInstances #-}
88{-# LANGUAGE GADTs #-}
99{-# LANGUAGE GeneralizedNewtypeDeriving #-}
10−{-# LANGUAGE ImplicitParams #-}
1110{-# LANGUAGE MultiParamTypeClasses #-}
1211{-# LANGUAGE OverloadedStrings #-}
1312{-# LANGUAGE RankNTypes #-}
@@ -23,44 +22,53 @@
2322module Katip.Core where
2423
2524-------------------------------------------------------------------------------
26−import Control.Applicative as A
25+import Control.Applicative as A
2726import Control.AutoUpdate
2827import Control.Concurrent
29−import qualified Control.Concurrent.Async as Async
28+import qualified Control.Concurrent.Async as Async
3029import Control.Concurrent.STM
31−import qualified Control.Concurrent.STM.TBQueue as BQ
30+import qualified Control.Concurrent.STM.TBQueue as BQ
3231import Control.Exception.Safe
33−import Control.Monad (unless, void)
32+import Control.Monad (unless, void, when)
3433import Control.Monad.Base
34+#if MIN_VERSION_base(4, 9, 0)
35+import qualified Control.Monad.Fail as MF
36+#endif
3537import Control.Monad.IO.Class
38+import Control.Monad.IO.Unlift
3639import Control.Monad.Trans.Class
3740import Control.Monad.Trans.Control
41+#if !MIN_VERSION_either(4, 5, 0)
3842import Control.Monad.Trans.Either
43+#endif
3944import Control.Monad.Trans.Except
4045import Control.Monad.Trans.Maybe
4146import Control.Monad.Trans.Reader
42−import Control.Monad.Trans.Resource (ResourceT, transResourceT)
43−import Control.Monad.Trans.State.Lazy (StateT, mapStateT)
44−import qualified Control.Monad.Trans.State.Strict as Strict (StateT, mapStateT)
45−import Control.Monad.Trans.Writer.Lazy (WriterT, mapWriterT)
46−import qualified Control.Monad.Trans.Writer.Strict as Strict (WriterT, mapWriterT)
47−import Control.Monad.Trans.RWS.Lazy (RWST, mapRWST)
48−import qualified Control.Monad.Trans.RWS.Strict as Strict (RWST, mapRWST)
49−import Data.Aeson (FromJSON (..), ToJSON (..),
50− object)
51−import qualified Data.Aeson as A
52−import Data.Foldable as FT
53−import qualified Data.HashMap.Strict as HM
47+import Control.Monad.Trans.Resource (MonadResource, ResourceT,
48+ transResourceT)
49+import Control.Monad.Trans.RWS.Lazy (RWST, mapRWST)
50+import qualified Control.Monad.Trans.RWS.Strict as Strict (RWST, mapRWST)
51+import Control.Monad.Trans.State.Lazy (StateT, mapStateT)
52+import qualified Control.Monad.Trans.State.Strict as Strict (StateT, mapStateT)
53+import Control.Monad.Trans.Writer.Lazy (WriterT, mapWriterT)
54+import qualified Control.Monad.Trans.Writer.Strict as Strict (WriterT,
55+ mapWriterT)
56+import Data.Aeson (FromJSON (..), ToJSON (..),
57+ object)
58+import qualified Data.Aeson as A
59+import Data.Foldable as FT
60+import qualified Data.HashMap.Strict as HM
5461import Data.List
55−import qualified Data.Map.Strict as M
56−import Data.Semigroup
62+import qualified Data.Map.Strict as M
63+import Data.Maybe (fromMaybe)
64+import Data.Semigroup as SG
5765import Data.String
5866import Data.String.Conv
59−import Data.Text (Text)
60−import qualified Data.Text as T
61−import qualified Data.Text.Lazy.Builder as B
67+import Data.Text (Text)
68+import qualified Data.Text as T
69+import qualified Data.Text.Lazy.Builder as B
6270import Data.Time
63−import GHC.Generics hiding (to)
71+import GHC.Generics hiding (to)
6472#if MIN_VERSION_base(4, 8, 0)
6573#if !MIN_VERSION_base(4, 9, 0)
6674import GHC.SrcLoc
@@ -68,7 +76,7 @@ import GHC.SrcLoc
6876import GHC.Stack
6977#endif
7078import Language.Haskell.TH
71−import qualified Language.Haskell.TH.Syntax as TH
79+import qualified Language.Haskell.TH.Syntax as TH
7280import Lens.Micro
7381import Lens.Micro.TH
7482import Network.HostName
@@ -95,7 +103,7 @@ readMay s = case [x | (x,t) <- reads s, ("","") <- lex t] of
95103-- IsString/OverloadedStrings, so "foo" will result in Namespace
96104-- ["foo"].
97105newtype Namespace = Namespace { unNamespace :: [Text] }
98− deriving (Eq,Show,Read,Ord,Generic,ToJSON,FromJSON,Semigroup,Monoid)
106+ deriving (Eq,Show,Read,Ord,Generic,ToJSON,FromJSON,SG.Semigroup,Monoid)
99107
100108instance IsString Namespace where
101109 fromString s = Namespace [fromString s]
@@ -135,7 +143,7 @@ data Severity
135143-- - 'V3' implies the maximum amount of payload information.
136144-- - Anything in between is left to the discretion of the developer.
137145data Verbosity = V0 | V1 | V2 | V3
138− deriving (Eq, Ord, Show, Read, Generic, Enum)
146+ deriving (Eq, Ord, Show, Read, Generic, Enum, Bounded)
139147
140148
141149-------------------------------------------------------------------------------
@@ -176,11 +184,26 @@ instance FromJSON Severity where
176184 Just x -> return x
177185 Nothing -> fail $ "Invalid Severity " ++ toS t
178186
187+instance ToJSON Verbosity where
188+ toJSON s = A.String $ case s of
189+ V0 -> "V0"
190+ V1 -> "V1"
191+ V2 -> "V2"
192+ V3 -> "V3"
193+
194+instance FromJSON Verbosity where
195+ parseJSON = A.withText "Verbosity" $ \s -> case s of
196+ "V0" -> return V0
197+ "V1" -> return V1
198+ "V2" -> return V2
199+ "V3" -> return V3
200+ _ -> fail $ "Invalid Verbosity " ++ toS s
201+
179202
180203-------------------------------------------------------------------------------
181204-- | Log message with Builder underneath; use '<>' to concat in O(1).
182205newtype LogStr = LogStr { unLogStr :: B.Builder }
183− deriving (Generic, Show)
206+ deriving (Generic, Show, Eq)
184207
185208instance IsString LogStr where
186209 fromString = LogStr . B.fromString
@@ -227,7 +250,9 @@ newtype ThreadIdText = ThreadIdText {
227250
228251
229252mkThreadIdText :: ThreadId -> ThreadIdText
230−mkThreadIdText = ThreadIdText . T.pack . show
253+mkThreadIdText = ThreadIdText . stripPrefix' "ThreadId " . T.pack . show
254+ where
255+ stripPrefix' pfx t = fromMaybe t (T.stripPrefix pfx t)
231256
232257
233258-------------------------------------------------------------------------------
@@ -247,31 +272,62 @@ data Item a = Item {
247272 } deriving (Generic, Functor)
248273makeLenses ''Item
249274
275+-- Manual instance because 'Loc' has no 'Eq' and 'Show' instances in old
276+-- versions of template-haskell (< 2.10)
277+instance Eq a => Eq (Item a) where
278+ a == b = FT.and [ _itemApp a == _itemApp b
279+ , _itemEnv a == _itemEnv b
280+ , _itemSeverity a == _itemSeverity b
281+ , _itemThread a == _itemThread b
282+ , _itemHost a == _itemHost b
283+ , _itemProcess a == _itemProcess b
284+ , _itemPayload a == _itemPayload b
285+ , _itemMessage a == _itemMessage b
286+ , _itemTime a == _itemTime b
287+ , _itemNamespace a == _itemNamespace b
288+ , case (_itemLoc a, _itemLoc b) of
289+ (Nothing, Nothing) -> True
290+ (Just l1, Just l2) -> FT.and [ loc_filename l1 == loc_filename l2
291+ , loc_package l1 == loc_package l2
292+ , loc_module l1 == loc_module l2
293+ , loc_start l1 == loc_start l2
294+ , loc_end l1 == loc_end l2
295+ ]
296+ _ -> False
297+ ]
250298
251299instance Show a => Show (Item a) where
252− show Item{..} = "Item {_itemApp = " ++ show _itemApp ++ ", " ++
253− "_itemEnv = " ++ show _itemEnv ++ ", " ++
254− "_itemSeverity = " ++ show _itemSeverity ++ ", " ++
255− "_itemThread = " ++ show _itemThread ++ ", " ++
256− "_itemHost = " ++ show _itemHost ++ ", " ++
257− "_itemProcess = " ++ show _itemProcess ++ ", " ++
258− "_itemPayload = " ++ show _itemPayload ++ ", " ++
259− "_itemMessage = " ++ show _itemMessage ++ ", " ++
260− "_itemTime = " ++ show _itemTime ++ ", " ++
261− "_itemNamespace = " ++ show _itemNamespace ++ ", " ++
262− "_itemLoc = " ++ show (LocShow <$> _itemLoc) ++ "}"
263−
300+ showsPrec d Item{..} = showParen (d >= 11) ( showString "Item {"
301+ . field "_itemApp" _itemApp
302+ . field "_itemEnv" _itemEnv
303+ . field "_itemSeverity" _itemSeverity
304+ . field "_itemThread" _itemThread
305+ . field "_itemHost" _itemHost
306+ . field "_itemProcess" _itemProcess
307+ . field "_itemPayload" _itemPayload
308+ . field "_itemMessage" _itemMessage
309+ . field "_itemTime" _itemTime
310+ . field "_itemNamespace" _itemNamespace
311+ . showString "_itemLoc = " . shows (LocShow <$> _itemLoc)
312+ . showChar '}'
313+ )
314+ where
315+ field n v = showString n . showString " = " . shows v . showString ", "
264316
265317newtype LocShow = LocShow Loc
266318
267319
268320instance Show LocShow where
269− show (LocShow Loc{..}) =
270− "Loc {loc_filename = " ++ show loc_filename ++ ", " ++
271− "loc_package = " ++ show loc_package ++ ", " ++
272− "loc_module = " ++ show loc_module ++ ", " ++
273− "loc_start = " ++ show loc_start ++ ", " ++
274− "loc_end = " ++ show loc_end ++ "}"
321+ showsPrec d (LocShow Loc{..}) = showParen (d >= 11) ( showString "Loc {"
322+ . field "loc_filename" loc_filename
323+ . field "loc_package" loc_package
324+ . field "loc_module" loc_module
325+ . field "loc_start" loc_start
326+ . showString "loc_end = " . shows loc_end
327+ . showChar '}'
328+ )
329+ where
330+ field n v = showString n . showString " = " . shows v . showString ", "
275331
276332
277333instance ToJSON a => ToJSON (Item a) where
@@ -484,52 +540,93 @@ itemJson verb a = toJSON $ a & itemPayload %~ payloadObject verb
484540--
485541-- Scribes should always take a 'Severity' and 'Verbosity'.
486542--
487−-- Severity is used to *exclude log messages* that are < the provided
488−-- Severity. For instance, if the user passes InfoS, DebugS items
489−-- should be ignored. Katip provides the 'permitItem' utility for this.
543+-- Severity is used to __exclude log messages__ that are lower than
544+-- the provided Severity. For instance, if the user passes InfoS,
545+-- DebugS items should be ignored. Katip provides the 'permitItem'
546+-- utility for this. The user or the scribe may use 'permitAND' and
547+-- 'permitOR' to further customize this filtering, even dynamically if
548+-- they wish to.
490549--
491550-- Verbosity is used to select keys from the log item's payload. Each
492551-- 'LogItem' instance describes what keys should be retained for each
493552-- Verbosity level. Use the 'payloadObject' utility for extracting the keys
494−-- that should be permitted.
553+-- that should be written.
495554--
496−-- There is no built-in mechanism in katip for telling a scribe that
497−-- its time to shut down. 'unregisterScribe' merely drops it from the
498−-- 'LogEnv'. This means there are 2 ways to handle resources as a scribe:
555+-- Scribes provide a finalizer IO action ('scribeFinalizer') that is
556+-- meant to synchronously flush any remaining writes and clean up any
557+-- resources acquired when the scribe was created. Internally, katip
558+-- keeps a buffer for each scribe's writes. When 'closeScribe' or
559+-- 'closeScribes' is called, that buffer stops accepting new log
560+-- messages and after the last item in its buffer is sent to 'liPush',
561+-- calls the finalizer. Thus, when the finalizer returns, katip can
562+-- assume that all resources are cleaned up and all log messages are
563+-- durably written.
499564--
500−-- 1. Pass in the resource when the scribe is created. Handle
501−-- allocation and release of the resource elsewhere. This is what the
502−-- Handle scribe does.
503−--
504−-- 2. Return a finalizing function that tells the scribe to shut
505−-- down. @katip-elasticsearch@'s @mkEsScribe@ returns an @IO (Scribe,
506−-- IO ())@. The finalizer will flush any queued log messages and shut
507−-- down gracefully before returning. This can be hooked into your
508−-- application's shutdown routine to ensure you never miss any log
509−-- messages on shutdown.
565+-- While katip internally buffers messages per 'ScribeSettings', it
566+-- sends them one at a time to the scribe. Depending on the scribe
567+-- itself, it may make sense for that scribe to keep its own internal
568+-- buffer to batch-send logs if writing items one at a time is not
569+-- efficient. The scribe implementer must be sure that on
570+-- finalization, all writes are committed synchronously.
571+
572+-- | Signature of a function passed to `Scribe` constructor and
573+-- mkScribe* functions that decides which messages to be
574+-- logged. Typically filters based on 'Severity', but can be
575+-- combined with other, custom logic with 'permitAND' and 'permitOR'
576+type PermitFunc = forall a. Item a -> IO Bool
577+
578+
579+-- | AND together 2 permit functions
580+permitAND :: PermitFunc -> PermitFunc -> PermitFunc
581+permitAND f1 f2 = \a -> liftA2 (&&) (f1 a) (f2 a)
582+
583+-- | OR together 2 permit functions
584+permitOR :: PermitFunc -> PermitFunc -> PermitFunc
585+permitOR f1 f2 = \a -> liftA2 (||) (f1 a) (f2 a)
586+
587+
510588data Scribe = Scribe {
511− liPush :: forall a. LogItem a => Item a -> IO ()
512− , scribeFinalizer :: IO ()
513− -- ^ Provide a *blocking* finalizer to call when your scribe is
514− -- removed. If this is not relevant to your scribe, return () is
515− -- fine.
589+ liPush :: forall a. LogItem a => Item a -> IO ()
590+ -- ^ How do we write an item to the scribe's output?
591+ , scribeFinalizer :: IO ()
592+ -- ^ Provide a __blocking__ finalizer to call when your scribe is
593+ -- removed. All pending writes should be flushed synchronously. If
594+ -- this is not relevant to your scribe, return () is fine.
595+ , scribePermitItem :: PermitFunc
596+ -- ^ Provide a filtering function to allow the item to be logged,
597+ -- or not. It can check Severity or some string in item's
598+ -- body. The initial value of this is usually created from
599+ -- 'permitItem'. Scribes and users can customize this by ANDing
600+ -- or ORing onto the default with 'permitAND' or 'permitOR'
516601 }
517602
518603
604+whenM :: Monad m => m Bool -> m () -> m ()
605+whenM mbool = (>>=) mbool . flip when
606+
607+
608+-- | Combine two scribes. Publishes to the left scribe if the left
609+-- would permit the item and to the right scribe if the right would
610+-- permit the item. Finalizers are called in sequence from left to
611+-- right.
519612instance Semigroup Scribe where
520− (Scribe pushA finA) <> (Scribe pushB finB) =
521− Scribe (\item -> pushA item >> pushB item) (finA `finally` finB)
613+ (Scribe pushA finA permitA) <> (Scribe pushB finB permitB) =
614+ Scribe (\item -> whenM (permitA item) (pushA item)
615+ >> whenM (permitB item) (pushB item)
616+ )
617+ (finA `finally` finB)
618+ (permitOR permitA permitB)
522619
523620
524621instance Monoid Scribe where
525− mempty = Scribe (const (return ())) (return ())
622+ mempty = Scribe (const (return ())) (return ()) (permitItem DebugS)
526623 mappend = (<>)
527624
528625
529626-------------------------------------------------------------------------------
530627data ScribeHandle = ScribeHandle {
531628 shScribe :: Scribe
532− , shChan :: BQ.TBQueue WorkerMessage
629+ , shChan :: BQ.TBQueue WorkerMessage
533630 }
534631
535632
@@ -541,8 +638,9 @@ data WorkerMessage where
541638
542639-------------------------------------------------------------------------------
543640-- | Should this item be logged given the user's maximum severity?
544−permitItem :: Severity -> Item a -> Bool
545−permitItem sev i = _itemSeverity i >= sev
641+-- Most new scribes will use this as a base for their 'PermitFunc'
642+permitItem :: Monad m => Severity -> Item a -> m Bool
643+permitItem sev item = return (_itemSeverity item >= sev)
546644
547645
548646-------------------------------------------------------------------------------
@@ -567,10 +665,10 @@ makeLenses ''LogEnv
567665
568666
569667-------------------------------------------------------------------------------
570−-- | Create a reasonable default InitLogEnv. Uses an 'AutoUdate' with
571−-- the default settings as the timer. If you are concerned about
572−-- timestamp precision or event ordering in log outputs like
573−-- ElasticSearch, you should replace the timer with 'getCurrentTime'
668+-- | Create a reasonable default InitLogEnv. Uses an 'AutoUpdate' which
669+-- updates the timer every 1ms. If you need even more timestamp
670+-- precision at the cost of performance, consider setting
671+-- '_logEnvTimer' with 'getCurrentTime'.
574672initLogEnv
575673 :: Namespace
576674 -- ^ A base namespace for this application
@@ -582,13 +680,15 @@ initLogEnv an env = LogEnv
582680 <*> getProcessID
583681 <*> pure an
584682 <*> pure env
585− <*> mkAutoUpdate defaultUpdateSettings { updateAction = getCurrentTime }
683+ <*> mkAutoUpdate defaultUpdateSettings { updateAction = getCurrentTime, updateFreq = 1000 }
586684 <*> pure mempty
587685
588686
589687-------------------------------------------------------------------------------
590688-- | Add a scribe to the list. All future log calls will go to this
591−-- scribe in addition to the others.
689+-- scribe in addition to the others. Writes will be buffered per the
690+-- ScribeSettings to prevent slow scribes from slowing down
691+-- logging. Writes will be dropped if the buffer fills.
592692registerScribe
593693 :: Text
594694 -- ^ Name the scribe
@@ -597,7 +697,7 @@ registerScribe
597697 -> LogEnv
598698 -> IO LogEnv
599699registerScribe nm scribe ScribeSettings {..} le = do
600− queue <- atomically (BQ.newTBQueue _scribeBufferSize)
700+ queue <- atomically (BQ.newTBQueue (fromIntegral _scribeBufferSize))
601701 worker <- spawnScribeWorker scribe queue
602702 let fin = do
603703 atomically (BQ.writeTBQueue queue PoisonPill)
@@ -612,7 +712,7 @@ registerScribe nm scribe ScribeSettings {..} le = do
612712
613713-------------------------------------------------------------------------------
614714spawnScribeWorker :: Scribe -> BQ.TBQueue WorkerMessage -> IO (Async.Async ())
615−spawnScribeWorker (Scribe write _) queue = Async.async go
715+spawnScribeWorker (Scribe write _ _) queue = Async.async go
616716 where
617717 go = do
618718 newCmd <- atomically (BQ.readTBQueue queue)
@@ -629,6 +729,7 @@ spawnScribeWorker (Scribe write _) queue = Async.async go
629729data ScribeSettings = ScribeSettings {
630730 _scribeBufferSize :: Int
631731 }
732+ deriving (Show, Eq)
632733
633734makeLenses ''ScribeSettings
634735
@@ -640,7 +741,7 @@ defaultScribeSettings = ScribeSettings 4096
640741
641742
642743-------------------------------------------------------------------------------
643−-- | Remove a scribe from the environment. This does *not* finalize
744+-- | Remove a scribe from the environment. This does __not__ finalize
644745-- the scribe. This mainly only makes sense to use with something like
645746-- MonadReader's @local@ function to temporarily disavow a single
646747-- logger for a block of code.
@@ -653,7 +754,7 @@ unregisterScribe nm = logEnvScribes %~ M.delete nm
653754
654755
655756-------------------------------------------------------------------------------
656−-- | Unregister *all* scribes. Note that this is *not* for closing or
757+-- | Unregister __all__ scribes. Note that this is __not__ for closing or
657758-- finalizing scribes, use 'closeScribes' for that. This mainly only
658759-- makes sense to use with something like MonadReader's @local@
659760-- function to temporarily disavow any loggers for a block of code.
@@ -665,9 +766,10 @@ clearScribes = logEnvScribes .~ mempty
665766
666767-------------------------------------------------------------------------------
667768-- | Finalize a scribe. The scribe is removed from the environment,
668−-- its finalizer is called and it can never be written to again. Note
669−-- that this will throw any exceptions yoru finalizer will throw, and
670−-- that LogEnv is immutable, so it will not be removed in that case.
769+-- its finalizer is called so that it can never be written to again
770+-- and all pending writes are flushed. Note that this will throw any
771+-- exceptions yoru finalizer will throw, and that LogEnv is immutable,
772+-- so it will not be removed in that case.
671773closeScribe
672774 :: Text
673775 -- ^ Name of the scribe
@@ -720,9 +822,11 @@ instance Katip m => Katip (ReaderT s m) where
720822 localLogEnv = mapReaderT . localLogEnv
721823
722824
825+#if !MIN_VERSION_either(4, 5, 0)
723826instance Katip m => Katip (EitherT s m) where
724827 getLogEnv = lift getLogEnv
725828 localLogEnv = mapEitherT . localLogEnv
829+#endif
726830
727831
728832instance Katip m => Katip (ExceptT s m) where
@@ -771,10 +875,13 @@ instance (Katip m) => Katip (ResourceT m) where
771875
772876
773877-------------------------------------------------------------------------------
774−-- | A concrete monad you can use to run logging actions.
878+-- | A concrete monad you can use to run logging actions. Use this if
879+-- you prefer an explicit monad transformer stack and adding layers as
880+-- opposed to implementing 'Katip' for your monad.
775881newtype KatipT m a = KatipT { unKatipT :: ReaderT LogEnv m a }
776882 deriving ( Functor, Applicative, Monad, MonadIO
777− , MonadMask, MonadCatch, MonadThrow, MonadTrans, MonadBase b)
883+ , MonadMask, MonadCatch, MonadThrow, MonadResource, MonadTrans
884+ , MonadBase b)
778885
779886
780887instance MonadIO m => Katip (KatipT m) where
@@ -795,6 +902,21 @@ instance (MonadBaseControl b m) => MonadBaseControl b (KatipT m) where
795902 liftBaseWith = defaultLiftBaseWith
796903 restoreM = defaultRestoreM
797904
905+instance MonadUnliftIO m => MonadUnliftIO (KatipT m) where
906+#if MIN_VERSION_unliftio_core(0, 2, 0)
907+ withRunInIO inner = KatipT $ ReaderT $ \le -> withRunInIO $ \run ->
908+ inner (run . runKatipT le)
909+#else
910+ askUnliftIO = KatipT $
911+ withUnliftIO $ \u ->
912+ pure (UnliftIO (unliftIO u . unKatipT))
913+#endif
914+
915+#if MIN_VERSION_base(4, 9, 0)
916+instance MF.MonadFail m => MF.MonadFail (KatipT m) where
917+ fail msg = lift (MF.fail msg)
918+ {-# INLINE fail #-}
919+#endif
798920
799921-------------------------------------------------------------------------------
800922-- | Execute 'KatipT' on a log env.
@@ -826,21 +948,32 @@ logItem
826948 -> m ()
827949logItem a ns loc sev msg = do
828950 LogEnv{..} <- getLogEnv
829− liftIO $ do
830− item <- Item
831− <$> pure _logEnvApp
832− <*> pure _logEnvEnv
833− <*> pure sev
834− <*> (mkThreadIdText <$> myThreadId)
835− <*> pure _logEnvHost
836− <*> pure _logEnvPid
837− <*> pure a
838− <*> pure msg
839− <*> _logEnvTimer
840− <*> pure (_logEnvApp <> ns)
841− <*> pure loc
842− FT.forM_ (M.elems _logEnvScribes) $ \ ScribeHandle {..} -> atomically (tryWriteTBQueue shChan (NewItem item))
843−
951+ logKatipItem =<< liftIO
952+ (Item <$> pure _logEnvApp
953+ <*> pure _logEnvEnv
954+ <*> pure sev
955+ <*> (mkThreadIdText <$> myThreadId)
956+ <*> pure _logEnvHost
957+ <*> pure _logEnvPid
958+ <*> pure a
959+ <*> pure msg
960+ <*> _logEnvTimer
961+ <*> pure (_logEnvApp <> ns)
962+ <*> pure loc)
963+
964+-- | Log already constructed 'Item'. This is the lowest level function that other log*
965+-- functions use.
966+-- It can be useful when implementing centralised logging services.
967+logKatipItem
968+ :: (A.Applicative m, LogItem a, Katip m)
969+ => Item a
970+ -> m ()
971+logKatipItem item = do
972+ LogEnv{..} <- getLogEnv
973+ liftIO $
974+ FT.forM_ (M.elems _logEnvScribes) $ \ ScribeHandle {..} -> do
975+ whenM (scribePermitItem shScribe item) $
976+ void $ atomically (tryWriteTBQueue shChan (NewItem item))
844977
845978-------------------------------------------------------------------------------
846979tryWriteTBQueue
@@ -853,6 +986,7 @@ tryWriteTBQueue q a = do
853986 unless full (writeTBQueue q a)
854987 return (not full)
855988
989+
856990-------------------------------------------------------------------------------
857991-- | Log with full context, but without any code location.
858992logF
@@ -885,7 +1019,7 @@ logException
8851019logException a ns sev action = action `catchAny` \e -> f e >> throwM e
8861020 where
8871021 f e = logF a ns sev (msg e)
888− msg e = ls (T.pack "An exception has occured: ") <> showLS e
1022+ msg e = ls (T.pack "An exception has occurred: ") <> showLS e
8891023
8901024
8911025-------------------------------------------------------------------------------
@@ -939,11 +1073,15 @@ liftLoc (Loc a b c (d1, d2) (e1, e2)) = [|Loc
9391073-- fill the 'Maybe Loc' gap in 'logF' of this module, and relies on implicit
9401074-- callstacks when available (GHC > 7.8).
9411075#if MIN_VERSION_base(4, 8, 0)
942−getLoc :: (?loc :: CallStack) => Maybe Loc
943−getLoc = case getCallStack ?loc of
1076+getLoc :: HasCallStack => Maybe Loc
1077+getLoc = case getCallStack callStack of
9441078 [] -> Nothing
945− xs -> Just . toLoc . last $ xs
1079+ xs -> Just . toLoc . head $ filter filterKatip xs
9461080 where
1081+ filterKatip :: (String, SrcLoc) -> Bool
1082+ filterKatip (_, srcloc) = not $
1083+ "katip-" `isPrefixOf` srcLocPackage srcloc
1084+
9471085 toLoc :: (String, SrcLoc) -> Loc
9481086 toLoc (_, l) = Loc {
9491087 loc_filename = srcLocFile l
@@ -974,7 +1112,7 @@ logT = [| \ a ns sev msg -> logItem a ns (Just $(getLocTH)) sev msg |]
9741112
9751113
9761114-------------------------------------------------------------------------------
977−-- | 'Loc'-tagged logging using implicit-callstacks when available.
1115+-- | 'Loc'-tagged logging using 'GHC.Stack' when available.
9781116--
9791117-- This function does not require template-haskell as it
9801118-- automatically uses <https://hackage.haskell.org/package/base-4.8.2.0/docs/GHC-Stack.html#v:getCallStack implicit-callstacks>
@@ -985,7 +1123,7 @@ logT = [| \ a ns sev msg -> logItem a ns (Just $(getLocTH)) sev msg |]
9851123--
9861124-- @logLoc obj mempty InfoS "Hello world"@
9871125#if MIN_VERSION_base(4, 8, 0)
988−logLoc :: (Applicative m, LogItem a, Katip m, ?loc :: CallStack)
1126+logLoc :: (Applicative m, LogItem a, Katip m, HasCallStack)
9891127#else
9901128logLoc :: (Applicative m, LogItem a, Katip m)
9911129#endif
katip/src/Katip/Monadic.hs+120−25
@@ -34,6 +34,8 @@ module Katip.Monadic
3434 , katipAddNamespace
3535 , katipAddContext
3636 , KatipContextTState(..)
37+ , NoLoggingT (..)
38+ , askLoggerIO
3739 ) where
3840
3941
@@ -42,16 +44,22 @@ import Control.Applicative
4244import Control.Exception.Safe
4345import Control.Monad.Base
4446import Control.Monad.Error.Class
47+#if MIN_VERSION_base(4, 9, 0)
48+import qualified Control.Monad.Fail as MF
49+#endif
4550import Control.Monad.IO.Class
51+import Control.Monad.IO.Unlift
4652import Control.Monad.Reader
4753import Control.Monad.State
4854import Control.Monad.Trans.Control
55+#if !MIN_VERSION_either(4, 5, 0)
4956import Control.Monad.Trans.Either (EitherT, mapEitherT)
57+#endif
5058import Control.Monad.Trans.Except (ExceptT, mapExceptT)
5159import Control.Monad.Trans.Identity (IdentityT, mapIdentityT)
52−import Control.Monad.Trans.List (ListT, mapListT)
5360import Control.Monad.Trans.Maybe (MaybeT, mapMaybeT)
54−import Control.Monad.Trans.Resource (ResourceT, transResourceT)
61+import Control.Monad.Trans.Resource (MonadResource,
62+ ResourceT, transResourceT)
5563import Control.Monad.Trans.RWS (RWST, mapRWST)
5664import qualified Control.Monad.Trans.RWS.Strict as Strict (RWST, mapRWST)
5765import qualified Control.Monad.Trans.State.Strict as Strict (StateT, mapStateT)
@@ -64,6 +72,12 @@ import qualified Data.HashMap.Strict as HM
6472import Data.Semigroup as Semi
6573import Data.Sequence as Seq
6674import Data.Text (Text)
75+#if MIN_VERSION_base(4, 8, 0)
76+#if !MIN_VERSION_base(4, 9, 0)
77+import GHC.SrcLoc
78+#endif
79+import GHC.Stack
80+#endif
6781import Language.Haskell.TH
6882-------------------------------------------------------------------------------
6983import Katip.Core
@@ -135,7 +149,7 @@ class Katip m => KatipContext m where
135149 localKatipContext :: (LogContexts -> LogContexts) -> m a -> m a
136150 getKatipNamespace :: m Namespace
137151 -- | Temporarily modify the current namespace for the duration of the
138− -- supplied monad. Used in 'katipAddContext'
152+ -- supplied monad. Used in 'katipAddNamespace'
139153 localKatipNamespace :: (Namespace -> Namespace) -> m a -> m a
140154
141155instance (KatipContext m, Katip (IdentityT m)) => KatipContext (IdentityT m) where
@@ -152,18 +166,13 @@ instance (KatipContext m, Katip (MaybeT m)) => KatipContext (MaybeT m) where
152166 localKatipNamespace = mapMaybeT . localKatipNamespace
153167
154168
169+#if !MIN_VERSION_either(4, 5, 0)
155170instance (KatipContext m, Katip (EitherT e m)) => KatipContext (EitherT e m) where
156171 getKatipContext = lift getKatipContext
157172 localKatipContext = mapEitherT . localKatipContext
158173 getKatipNamespace = lift getKatipNamespace
159174 localKatipNamespace = mapEitherT . localKatipNamespace
160−
161−
162−instance (KatipContext m, Katip (ListT m)) => KatipContext (ListT m) where
163− getKatipContext = lift getKatipContext
164− localKatipContext = mapListT . localKatipContext
165− getKatipNamespace = lift getKatipNamespace
166− localKatipNamespace = mapListT . localKatipNamespace
175+#endif
167176
168177
169178instance (KatipContext m, Katip (ReaderT r m)) => KatipContext (ReaderT r m) where
@@ -234,9 +243,9 @@ deriving instance (Monad m, KatipContext m) => KatipContext (KatipT m)
234243-------------------------------------------------------------------------------
235244-- | Log with everything, including a source code location. This is
236245-- very low level and you typically can use 'logTM' in its
237−-- place. Automaticallysupplies payload and namespace.
246+-- place. Automatically supplies payload and namespace.
238247logItemM
239− :: (Applicative m, KatipContext m)
248+ :: (Applicative m, KatipContext m, HasCallStack)
240249 => Maybe Loc
241250 -> Severity
242251 -> LogStr
@@ -273,20 +282,21 @@ logTM = [| logItemM (Just $(getLocTH)) |]
273282
274283
275284-------------------------------------------------------------------------------
276−-- | 'Loc'-tagged logging when using template-haskell. Automatically
277−-- supplies payload and namespace.
285+-- | 'Loc'-tagged logging when using 'GHC.Stack.getCallStack' implicit-callstacks>.
286+-- Automatically supplies payload and namespace.
278287--
279288-- Same consideration as `logLoc` applies.
280289--
281−-- This function does not require template-haskell as it
282−-- automatically uses <https://hackage.haskell.org/package/base-4.8.2.0/docs/GHC-Stack.html#v:getCallStack implicit-callstacks>
283−-- when the code is compiled using GHC > 7.8. Using an older version of the
284−-- compiler will result in the emission of a log line without any location information,
285−-- so be aware of it. Users using GHC <= 7.8 may want to use the template-haskell function
290+-- Location will be logged from the module that invokes 'logLocM' so
291+-- be aware that wrapping 'logLocM' will make location reporting useless.
292+--
293+-- This function does not require template-haskell. Using GHC <= 7.8 will result
294+-- in the emission of a log line without any location information.
295+-- Users using GHC <= 7.8 may want to use the template-haskell function
286296-- `logTM` for maximum compatibility.
287297--
288298-- @logLocM InfoS "Hello world"@
289−logLocM :: (Applicative m, KatipContext m)
299+logLocM :: (Applicative m, KatipContext m, HasCallStack)
290300 => Severity
291301 -> LogStr
292302 -> m ()
@@ -306,15 +316,16 @@ logExceptionM
306316logExceptionM action sev = action `catchAny` \e -> f e >> throwM e
307317 where
308318 f e = logFM sev (msg e)
309− msg e = ls ("An exception has occured: " :: Text) Semi.<> showLS e
319+ msg e = ls ("An exception has occurred: " :: Text) Semi.<> showLS e
310320
311321
312322-------------------------------------------------------------------------------
313323-- | Provides a simple transformer that defines a 'KatipContext'
314−-- instance for a fixed namespace and context. You will typically only
315−-- use this if you are forced to run in IO but still want to have your
316−-- log context. This is the slightly more powerful version of KatipT
317−-- in that it provides KatipContext instead of just Katip. For instance:
324+-- instance for a fixed namespace and context. Just like 'KatipT', you
325+-- should use this if you prefer an explicit transformer stack and
326+-- don't want to (or cannot) define 'KatipContext' for your monad
327+-- . This is the slightly more powerful version of KatipT in that it
328+-- provides KatipContext instead of just Katip. For instance:
318329--
319330-- @
320331-- threadWithLogging = do
@@ -339,6 +350,7 @@ newtype KatipContextT m a = KatipContextT {
339350 , MonadWriter w
340351 , MonadError e
341352 , MonadPlus
353+ , MonadResource
342354 , Alternative
343355 , MonadFix
344356 , MonadTrans
@@ -384,6 +396,22 @@ instance (MonadIO m) => KatipContext (KatipContextT m) where
384396 getKatipNamespace = KatipContextT $ ReaderT $ \lts -> return (ltsNamespace lts)
385397 localKatipNamespace f (KatipContextT m) = KatipContextT $ local (\s -> s { ltsNamespace = f (ltsNamespace s)}) m
386398
399+instance MonadUnliftIO m => MonadUnliftIO (KatipContextT m) where
400+#if MIN_VERSION_unliftio_core(0, 2, 0)
401+ withRunInIO inner = KatipContextT $ ReaderT $ \lts -> withRunInIO $ \run ->
402+ inner (run . runKatipContextT (ltsLogEnv lts) (ltsContext lts) (ltsNamespace lts))
403+#else
404+ askUnliftIO = KatipContextT $
405+ withUnliftIO $ \u ->
406+ pure (UnliftIO (unliftIO u . unKatipContextT))
407+#endif
408+
409+
410+#if MIN_VERSION_base(4, 9, 0)
411+instance MF.MonadFail m => MF.MonadFail (KatipContextT m) where
412+ fail msg = lift (MF.fail msg)
413+ {-# INLINE fail #-}
414+#endif
387415
388416-------------------------------------------------------------------------------
389417runKatipContextT :: (LogItem c) => LogEnv -> c -> Namespace -> KatipContextT m a -> m a
@@ -424,3 +452,70 @@ katipAddContext
424452 -> m a
425453 -> m a
426454katipAddContext i = localKatipContext (<> (liftPayload i))
455+
456+newtype NoLoggingT m a = NoLoggingT {
457+ runNoLoggingT :: m a
458+ } deriving ( Functor
459+ , Applicative
460+ , Monad
461+ , MonadIO
462+ , MonadThrow
463+ , MonadCatch
464+ , MonadMask
465+ , MonadBase b
466+ , MonadState s
467+ , MonadWriter w
468+ , MonadError e
469+ , MonadPlus
470+ , Alternative
471+ , MonadFix
472+ , MonadReader r
473+ )
474+
475+instance MonadTrans NoLoggingT where
476+ lift = NoLoggingT
477+
478+instance MonadTransControl NoLoggingT where
479+ type StT NoLoggingT a = a
480+ liftWith f = NoLoggingT $ f runNoLoggingT
481+ restoreT = NoLoggingT
482+ {-# INLINE liftWith #-}
483+ {-# INLINE restoreT #-}
484+
485+instance MonadBaseControl b m => MonadBaseControl b (NoLoggingT m) where
486+ type StM (NoLoggingT m) a = StM m a
487+ liftBaseWith f = NoLoggingT $
488+ liftBaseWith $ \runInBase ->
489+ f $ runInBase . runNoLoggingT
490+ restoreM = NoLoggingT . restoreM
491+
492+instance MonadUnliftIO m => MonadUnliftIO (NoLoggingT m) where
493+#if MIN_VERSION_unliftio_core(0, 2, 0)
494+ withRunInIO inner = NoLoggingT $ withRunInIO $ \run ->
495+ inner (run . runNoLoggingT)
496+#else
497+ askUnliftIO = NoLoggingT $
498+ withUnliftIO $ \u ->
499+ pure (UnliftIO (unliftIO u . runNoLoggingT))
500+#endif
501+
502+
503+instance MonadIO m => Katip (NoLoggingT m) where
504+ getLogEnv = liftIO (initLogEnv "NoLoggingT" "no-logging")
505+ localLogEnv = const id
506+
507+instance MonadIO m => KatipContext (NoLoggingT m) where
508+ getKatipContext = pure mempty
509+ localKatipContext = const id
510+ getKatipNamespace = pure mempty
511+ localKatipNamespace = const id
512+
513+
514+-- | Convenience function for when you have to integrate with a third
515+-- party API that takes a generic logging function as an argument.
516+askLoggerIO :: (Applicative m, KatipContext m) => m (Severity -> LogStr -> IO ())
517+askLoggerIO = do
518+ ctx <- getKatipContext
519+ ns <- getKatipNamespace
520+ logEnv <- getLogEnv
521+ pure (\sev msg -> runKatipT logEnv $ logF ctx ns sev msg)
katip/src/Katip/Scribes/Handle.hs+100−30
@@ -1,15 +1,15 @@
1−{-# LANGUAGE RecordWildCards #-}
2−
31module Katip.Scribes.Handle where
42
53-------------------------------------------------------------------------------
64import Control.Applicative as A
75import Control.Concurrent
8−import Control.Exception (bracket_)
9−import Control.Monad
6+import Control.Exception (bracket_, finally)
107import Data.Aeson
8+import Data.Text.Lazy (toStrict)
9+import Data.Text.Lazy.Encoding (decodeUtf8)
1110import qualified Data.HashMap.Strict as HM
12−import Data.Monoid
11+import Data.Monoid as M
12+import Data.Scientific as S
1313import Data.Text (Text)
1414import Data.Text.Lazy.Builder
1515import Data.Text.Lazy.IO as T
@@ -22,7 +22,7 @@ import Katip.Format.Time (formatAsLogTime)
2222
2323-------------------------------------------------------------------------------
2424brackets :: Builder -> Builder
25−brackets m = fromText "[" <> m <> fromText "]"
25+brackets m = fromText "[" M.<> m <> fromText "]"
2626
2727
2828-------------------------------------------------------------------------------
@@ -34,10 +34,13 @@ getKeys verb a = concat (renderPair A.<$> HM.toList (payloadObject verb a))
3434 case v of
3535 Object o -> concat [renderPair (k <> "." <> k', v') | (k', v') <- HM.toList o]
3636 String t -> [fromText (k <> ":" <> t)]
37− Number n -> [fromText (k <> ":") <> fromString (show n)]
37+ Number n -> [fromText (k <> ":") <> fromString (formatNumber n)]
3838 Bool b -> [fromText (k <> ":") <> fromString (show b)]
3939 Null -> [fromText (k <> ":null")]
4040 _ -> mempty -- Can't think of a sensible way to handle arrays
41+ formatNumber :: Scientific -> String
42+ formatNumber n =
43+ formatScientific Generic (if isFloating n then Nothing else Just 0) n
4144
4245
4346-------------------------------------------------------------------------------
@@ -46,52 +49,119 @@ data ColorStrategy
4649 -- ^ Whether to use color control chars in log output
4750 | ColorIfTerminal
4851 -- ^ Color if output is a terminal
52+ deriving (Show, Eq)
4953
5054-------------------------------------------------------------------------------
5155-- | Logs to a file handle such as stdout, stderr, or a file. Contexts
5256-- and other information will be flattened out into bracketed
5357-- fields. For example:
5458--
55−-- > [2016-05-11 21:01:15][MyApp][Info][myhost.example.com][1724][ThreadId 1154][main:Helpers.Logging Helpers/Logging.hs:32:7] Started
56−-- > [2016-05-11 21:01:15][MyApp.confrabulation][Debug][myhost.example.com][1724][ThreadId 1154][confrab_factor:42.0][main:Helpers.Logging Helpers/Logging.hs:41:9] Confrabulating widgets, with extra namespace and context
57−-- > [2016-05-11 21:01:15][MyApp][Info][myhost.example.com][1724][ThreadId 1154][main:Helpers.Logging Helpers/Logging.hs:43:7] Namespace and context are back to normal
59+-- > [2016-05-11 21:01:15][MyApp][Info][myhost.example.com][PID 1724][ThreadId 1154][main:Helpers.Logging Helpers/Logging.hs:32:7] Started
60+-- > [2016-05-11 21:01:15][MyApp.confrabulation][Debug][myhost.example.com][PID 1724][ThreadId 1154][confrab_factor:42.0][main:Helpers.Logging Helpers/Logging.hs:41:9] Confrabulating widgets, with extra namespace and context
61+-- > [2016-05-11 21:01:15][MyApp][Info][myhost.example.com][PID 1724][ThreadId 1154][main:Helpers.Logging Helpers/Logging.hs:43:7] Namespace and context are back to normal
62+--
63+-- Returns the newly-created `Scribe`. The finalizer flushes the
64+-- handle. Handle mode is set to 'LineBuffering' automatically.
65+mkHandleScribe :: ColorStrategy -> Handle -> PermitFunc -> Verbosity -> IO Scribe
66+mkHandleScribe = mkHandleScribeWithFormatter bracketFormat
67+
68+-- | Logs to a file handle such as stdout, stderr, or a file. Takes a custom
69+-- `ItemFormatter` that can be used to format `Item` as needed.
5870--
59−-- Returns the newly-created `Scribe`. The finalizer flushes the handle.
60−mkHandleScribe :: ColorStrategy -> Handle -> Severity -> Verbosity -> IO Scribe
61−mkHandleScribe cs h sev verb = do
71+-- Returns the newly-created `Scribe`. The finalizer flushes the
72+-- handle. Handle mode is set to 'LineBuffering' automatically.
73+mkHandleScribeWithFormatter :: (forall a . LogItem a => ItemFormatter a)
74+ -> ColorStrategy
75+ -> Handle
76+ -> PermitFunc
77+ -> Verbosity
78+ -> IO Scribe
79+mkHandleScribeWithFormatter itemFormatter cs h permitF verb = do
6280 hSetBuffering h LineBuffering
6381 colorize <- case cs of
6482 ColorIfTerminal -> hIsTerminalDevice h
6583 ColorLog b -> return b
6684 lock <- newMVar ()
6785 let logger i@Item{..} = do
68− when (permitItem sev i) $ bracket_ (takeMVar lock) (putMVar lock ()) $
69− T.hPutStrLn h $ toLazyText $ formatItem colorize verb i
70− return $ Scribe logger (hFlush h)
86+ bracket_ (takeMVar lock) (putMVar lock ()) $
87+ T.hPutStrLn h $ toLazyText $ itemFormatter colorize verb i
88+ return $ Scribe logger (hFlush h) permitF
89+
90+
91+-------------------------------------------------------------------------------
92+-- | A specialization of 'mkHandleScribe' that takes a 'FilePath'
93+-- instead of a 'Handle'. It is responsible for opening the file in
94+-- 'AppendMode' and will close the file handle on
95+-- 'closeScribe'/'closeScribes'. Does not do log coloring. Sets handle
96+-- to 'LineBuffering' mode.
97+mkFileScribe :: FilePath -> PermitFunc -> Verbosity -> IO Scribe
98+mkFileScribe f permitF verb = do
99+ h <- openFile f AppendMode
100+ Scribe logger finalizer permit <- mkHandleScribe (ColorLog False) h permitF verb
101+ return (Scribe logger (finalizer `finally` hClose h) permit)
71102
72103
73104-------------------------------------------------------------------------------
74−formatItem :: LogItem a => Bool -> Verbosity -> Item a -> Builder
75−formatItem withColor verb Item{..} =
105+-- | A custom ItemFormatter for logging `Item`s. Takes a `Bool` indicating
106+-- whether to colorize the output, `Verbosity` of output, and an `Item` to
107+-- format.
108+--
109+-- See `bracketFormat` and `jsonFormat` for examples.
110+type ItemFormatter a = Bool -> Verbosity -> Item a -> Builder
111+
112+
113+formatItem :: LogItem a => ItemFormatter a
114+formatItem = bracketFormat
115+{-# DEPRECATED formatItem "Use bracketFormat instead" #-}
116+
117+-- | A traditional 'bracketed' log format. Contexts and other information will
118+-- be flattened out into bracketed fields. For example:
119+--
120+-- > [2016-05-11 21:01:15][MyApp][Info][myhost.example.com][PID 1724][ThreadId 1154][main:Helpers.Logging Helpers/Logging.hs:32:7] Started
121+-- > [2016-05-11 21:01:15][MyApp.confrabulation][Debug][myhost.example.com][PID 1724][ThreadId 1154][confrab_factor:42.0][main:Helpers.Logging Helpers/Logging.hs:41:9] Confrabulating widgets, with extra namespace and context
122+-- > [2016-05-11 21:01:15][MyApp][Info][myhost.example.com][PID 1724][ThreadId 1154][main:Helpers.Logging Helpers/Logging.hs:43:7] Namespace and context are back to normal
123+bracketFormat :: LogItem a => ItemFormatter a
124+bracketFormat withColor verb Item{..} =
76125 brackets nowStr <>
77126 brackets (mconcat $ map fromText $ intercalateNs _itemNamespace) <>
78127 brackets (fromText (renderSeverity' _itemSeverity)) <>
79128 brackets (fromString _itemHost) <>
80− brackets (fromString (show _itemProcess)) <>
81− brackets (fromText (getThreadIdText _itemThread)) <>
129+ brackets ("PID " <> fromString (show _itemProcess)) <>
130+ brackets ("ThreadId " <> fromText (getThreadIdText _itemThread)) <>
82131 mconcat ks <>
83132 maybe mempty (brackets . fromString . locationToString) _itemLoc <>
84133 fromText " " <> (unLogStr _itemMessage)
85134 where
86135 nowStr = fromText (formatAsLogTime _itemTime)
87136 ks = map brackets $ getKeys verb _itemPayload
88− renderSeverity' s = case s of
89− EmergencyS -> red $ renderSeverity s
90− AlertS -> red $ renderSeverity s
91− CriticalS -> red $ renderSeverity s
92− ErrorS -> red $ renderSeverity s
93− WarningS -> yellow $ renderSeverity s
94− _ -> renderSeverity s
137+ renderSeverity' severity =
138+ colorBySeverity withColor severity (renderSeverity severity)
139+
140+-- | Logs items as JSON. This can be useful in circumstances where you already
141+-- have infrastructure that is expecting JSON to be logged to a standard stream
142+-- or file. For example:
143+--
144+-- > {"at":"2018-10-02T21:50:30.4523848Z","env":"production","ns":["MyApp"],"data":{},"app":["MyApp"],"msg":"Started","pid":"10456","loc":{"loc_col":9,"loc_pkg":"main","loc_mod":"Helpers.Logging","loc_fn":"Helpers\\Logging.hs","loc_ln":44},"host":"myhost.example.com","sev":"Info","thread":"ThreadId 139"}
145+-- > {"at":"2018-10-02T21:50:30.4523848Z","env":"production","ns":["MyApp","confrabulation"],"data":{"confrab_factor":42},"app":["MyApp"],"msg":"Confrabulating widgets, with extra namespace and context","pid":"10456","loc":{"loc_col":11,"loc_pkg":"main","loc_mod":"Helpers.Logging","loc_fn":"Helpers\\Logging.hs","loc_ln":53},"host":"myhost.example.com","sev":"Debug","thread":"ThreadId 139"}
146+-- > {"at":"2018-10-02T21:50:30.4523848Z","env":"production","ns":["MyApp"],"data":{},"app":["MyApp"],"msg":"Namespace and context are back to normal","pid":"10456","loc":{"loc_col":9,"loc_pkg":"main","loc_mod":"Helpers.Logging","loc_fn":"Helpers\\Logging.hs","loc_ln":55},"host":"myhost.example.com","sev":"Info","thread":"ThreadId 139"}
147+jsonFormat :: LogItem a => ItemFormatter a
148+jsonFormat withColor verb i =
149+ fromText $
150+ colorBySeverity withColor (_itemSeverity i) $
151+ toStrict $ decodeUtf8 $ encode $ itemJson verb i
152+
153+-- | Color a text message based on `Severity`. `ErrorS` and more severe errors
154+-- are colored red, `WarningS` is colored yellow, and all other messages are
155+-- rendered in the default color.
156+colorBySeverity :: Bool -> Severity -> Text -> Text
157+colorBySeverity withColor severity msg = case severity of
158+ EmergencyS -> red msg
159+ AlertS -> red msg
160+ CriticalS -> red msg
161+ ErrorS -> red msg
162+ WarningS -> yellow msg
163+ _ -> msg
164+ where
95165 red = colorize "31"
96166 yellow = colorize "33"
97167 colorize c s
@@ -103,8 +173,8 @@ formatItem withColor verb Item{..} =
103173-- stdout. This is a decent example of how to build a LogEnv and is
104174-- best for scripts that just need a quick, reasonable set up to log
105175-- to stdout.
106−ioLogEnv :: Severity -> Verbosity -> IO LogEnv
107−ioLogEnv sev verb = do
176+ioLogEnv :: PermitFunc -> Verbosity -> IO LogEnv
177+ioLogEnv permit verb = do
108178 le <- initLogEnv "io" "io"
109− lh <- mkHandleScribe ColorIfTerminal stdout sev verb
179+ lh <- mkHandleScribe ColorIfTerminal stdout permit verb
110180 registerScribe "stdout" lh defaultScribeSettings le
katip/test/Katip/Tests.hs+48−9
@@ -19,12 +19,13 @@ import Control.Exception.Safe
1919import Data.Aeson
2020import qualified Data.HashMap.Strict as HM
2121import qualified Data.Map.Strict as M
22−import Data.Monoid
22+import Data.Monoid as Monoid
2323import Data.Text (Text)
2424import qualified Data.Text.Lazy.Builder as B
2525import Data.Time
2626import Data.Time.Clock.POSIX
2727import Language.Haskell.TH
28+import Lens.Micro (ASetter, (&), (.~))
2829import System.Posix.Types
2930import Test.QuickCheck.Instances ()
3031import Test.Tasty
@@ -41,6 +42,9 @@ tests = testGroup "Katip"
4142 [
4243 testProperty "JSON cycle Item" $ \(i :: Item ()) ->
4344 prop_json_cycle i
45+ , testProperty "JSON cycle verbosity" $ \(v :: Verbosity) ->
46+ prop_json_cycle v
47+ , eqItemTests
4448 , testProperty "renderSeverity/textToSeverity cycle" $ \sev ->
4549 textToSeverity(renderSeverity sev) === Just sev
4650 , testProperty "processIDToText/textToProcessID cycle" $ \pid ->
@@ -122,14 +126,22 @@ loggingTests = testGroup "logging"
122126 , (baseNS <> "added" <> "namespace", baseCtx, "with namespaces")
123127 , (baseNS <> "added", HM.insert "additional" (Bool True) baseCtx, "additional context")
124128 ]
129+ , testCase "Katip.Monadic.logLocM" $ do
130+ (le, items) <- recordingEnv
131+ runKatipContextT le (sl "base_context" (42 :: Int)) "base_namespace" $ logLocM InfoS "basic log"
132+ _ <- closeScribes le
133+ loggedItems <- readTVarIO items
134+ let loc = _itemLoc (head loggedItems)
135+ fmap loc_module loc @?= Just "Katip.Tests"
125136 ]
126137 where
127138 recordingEnv :: IO (LogEnv, TVar [Item Object])
128139 recordingEnv = do
129− items <- newTVarIO mempty
140+ items <- newTVarIO Monoid.mempty
130141 let scribe = Scribe
131142 { liPush = \i -> atomically (modifyTVar' items (<> [toObject <$> i]))
132143 , scribeFinalizer = return ()
144+ , scribePermitItem = permitItem DebugS
133145 }
134146 le1 <- initLogEnv "tests" "test"
135147 le2 <- registerScribe "recorder" scribe defaultScribeSettings le1
@@ -143,7 +155,7 @@ trivialScribe :: IO (Scribe, TVar Bool)
143155trivialScribe = do
144156 finalizerCalled <- newTVarIO False
145157 let finalizer = atomically (writeTVar finalizerCalled True)
146− return (Scribe (const (return ())) finalizer, finalizerCalled)
158+ return (Scribe (const (return ())) finalizer (permitItem DebugS), finalizerCalled)
147159
148160
149161-------------------------------------------------------------------------------
@@ -153,7 +165,7 @@ brokenScribe scribeNum = do
153165 let finalizer = do
154166 atomically (writeTVar finalizerCalled True)
155167 throw (ScribeBroken scribeNum)
156− return (Scribe (const (return ())) finalizer, finalizerCalled)
168+ return (Scribe (const (return ())) finalizer (permitItem DebugS), finalizerCalled)
157169
158170
159171-------------------------------------------------------------------------------
@@ -245,11 +257,6 @@ deriving instance Arbitrary Namespace
245257deriving instance Arbitrary Environment
246258deriving instance Arbitrary ThreadIdText
247259deriving instance Arbitrary CPid
248−#if !MIN_VERSION_base(4, 8, 0)
249−deriving instance Eq Loc
250−#endif
251−deriving instance Eq LogStr
252−deriving instance (Eq a) => Eq (Item a)
253260
254261
255262-------------------------------------------------------------------------------
@@ -262,6 +269,10 @@ instance Arbitrary Loc where
262269 return $ Loc f p m s s
263270
264271
272+-------------------------------------------------------------------------------
273+instance Arbitrary Verbosity where
274+ arbitrary = oneof $ map pure [V0, V1, V2, V3]
275+
265276-------------------------------------------------------------------------------
266277instance Arbitrary Severity where
267278 arbitrary = oneof $ map pure [ DebugS
@@ -278,3 +289,31 @@ instance Arbitrary Severity where
278289-------------------------------------------------------------------------------
279290instance Arbitrary LogStr where
280291 arbitrary = LogStr . B.fromText <$> arbitrary
292+
293+-------------------------------------------------------------------------------
294+-- Somewhat test whether all fields are taken into account in `==`
295+
296+#if !MIN_VERSION_template_haskell(2, 10, 0)
297+-- `testProperty` requires these instances
298+deriving instance Eq Loc
299+deriving instance Show Loc
300+#endif
301+
302+eqItemTests :: TestTree
303+eqItemTests = testGroup "Eq Item"
304+ [ testProperty "itemApp" $ prop_field itemApp
305+ , testProperty "itemEnv" $ prop_field itemEnv
306+ , testProperty "itemSeverity" $ prop_field itemSeverity
307+ , testProperty "itemThread" $ prop_field itemThread
308+ , testProperty "itemHost" $ prop_field itemHost
309+ , testProperty "itemProcess" $ prop_field itemProcess
310+ , testProperty "itemPayload" $ prop_field itemPayload
311+ , testProperty "itemMessage" $ prop_field itemMessage
312+ , testProperty "itemTime" $ prop_field itemTime
313+ , testProperty "itemNamespace" $ prop_field itemNamespace
314+ , testProperty "itemLoc" $ prop_field itemLoc
315+ ]
316+ where
317+ prop_field :: Eq a => ASetter (Item ()) (Item ()) a a -> Item () -> a -> a -> Bool
318+ prop_field field item f1 f2 =
319+ ((item & field .~ f1) == (item & field .~ f2)) == (f1 == f2)
katip/test/Katip/Tests/Scribes/Handle-json.goldennew file+38
@@ -0,0 +1,38 @@
1+{"at":"2016-06-12T12:34:56Z","env":"production","ns":["foo"],"data":{"note":{"deep":"some note"}},"app":["app"],"msg":"message","pid":"7331","loc":null,"host":"example","sev":"Debug","thread":"1337"}
2+{"at":"2016-06-12T12:34:56Z","env":"production","ns":["foo"],"data":{"note":{"deep":"some note"}},"app":["app"],"msg":"message","pid":"7331","loc":null,"host":"example","sev":"Info","thread":"1337"}
3+{"at":"2016-06-12T12:34:56Z","env":"production","ns":["foo"],"data":{"note":{"deep":"some note"}},"app":["app"],"msg":"message","pid":"7331","loc":null,"host":"example","sev":"Notice","thread":"1337"}
4+{"at":"2016-06-12T12:34:56Z","env":"production","ns":["foo"],"data":{"note":{"deep":"some note"}},"app":["app"],"msg":"message","pid":"7331","loc":null,"host":"example","sev":"Warning","thread":"1337"}
5+{"at":"2016-06-12T12:34:56Z","env":"production","ns":["foo"],"data":{"note":{"deep":"some note"}},"app":["app"],"msg":"message","pid":"7331","loc":null,"host":"example","sev":"Error","thread":"1337"}
6+{"at":"2016-06-12T12:34:56Z","env":"production","ns":["foo"],"data":{"note":{"deep":"some note"}},"app":["app"],"msg":"message","pid":"7331","loc":null,"host":"example","sev":"Critical","thread":"1337"}
7+{"at":"2016-06-12T12:34:56Z","env":"production","ns":["foo"],"data":{"note":{"deep":"some note"}},"app":["app"],"msg":"message","pid":"7331","loc":null,"host":"example","sev":"Alert","thread":"1337"}
8+{"at":"2016-06-12T12:34:56Z","env":"production","ns":["foo"],"data":{"note":{"deep":"some note"}},"app":["app"],"msg":"message","pid":"7331","loc":null,"host":"example","sev":"Emergency","thread":"1337"}
9+{"at":"2016-06-12T12:34:56Z","env":"production","ns":["foo"],"data":{"note":{"deep":"some note"}},"app":["app"],"msg":"message","pid":"7331","loc":null,"host":"example","sev":"Info","thread":"0"}
10+{"at":"2016-06-12T12:34:56Z","env":"production","ns":["foo"],"data":{"note":{"deep":"some note"}},"app":["app"],"msg":"message","pid":"7331","loc":null,"host":"example","sev":"Info","thread":"1"}
11+{"at":"2016-06-12T12:34:56Z","env":"production","ns":["foo"],"data":{"note":{"deep":"some note"}},"app":["app"],"msg":"message","pid":"7331","loc":null,"host":"example","sev":"Info","thread":"1337"}
12+{"at":"2016-06-12T12:34:56Z","env":"production","ns":["foo"],"data":{"note":{"deep":"some note"}},"app":["app"],"msg":"message","pid":"7331","loc":null,"host":"example","sev":"Info","thread":"2147483647"}
13+{"at":"2016-06-12T12:34:56Z","env":"production","ns":["foo"],"data":{"note":{"deep":"some note"}},"app":["app"],"msg":"message","pid":"7331","loc":null,"host":"example","sev":"Info","thread":"1337"}
14+{"at":"2016-06-12T12:34:56Z","env":"production","ns":["foo"],"data":{"note":{"deep":"some note"}},"app":["app"],"msg":"message","pid":"7331","loc":null,"host":"www.example.com","sev":"Info","thread":"1337"}
15+{"at":"2016-06-12T12:34:56Z","env":"production","ns":["foo"],"data":{"note":{"deep":"some note"}},"app":["app"],"msg":"message","pid":"7331","loc":null,"host":"127.0.0.1","sev":"Info","thread":"1337"}
16+{"at":"2016-06-12T12:34:56Z","env":"production","ns":["foo"],"data":{"note":{"deep":"some note"}},"app":["app"],"msg":"message","pid":"0","loc":null,"host":"example","sev":"Info","thread":"1337"}
17+{"at":"2016-06-12T12:34:56Z","env":"production","ns":["foo"],"data":{"note":{"deep":"some note"}},"app":["app"],"msg":"message","pid":"1","loc":null,"host":"example","sev":"Info","thread":"1337"}
18+{"at":"2016-06-12T12:34:56Z","env":"production","ns":["foo"],"data":{"note":{"deep":"some note"}},"app":["app"],"msg":"message","pid":"1337","loc":null,"host":"example","sev":"Info","thread":"1337"}
19+{"at":"2016-06-12T12:34:56Z","env":"production","ns":["foo"],"data":{"note":{"deep":"some note"}},"app":["app"],"msg":"message","pid":"2147483647","loc":null,"host":"example","sev":"Info","thread":"1337"}
20+{"at":"2016-06-12T12:34:56Z","env":"production","ns":["foo"],"data":{"note":{"deep":"some note"}},"app":["app"],"msg":"message","pid":"7331","loc":null,"host":"example","sev":"Info","thread":"1337"}
21+{"at":"2016-06-12T12:34:56Z","env":"production","ns":["foo"],"data":{"note":{"deep":"some note"}},"app":["app"],"msg":"message\nwith newline","pid":"7331","loc":null,"host":"example","sev":"Info","thread":"1337"}
22+{"at":"2016-06-12T12:34:56Z","env":"production","ns":["foo"],"data":{"note":{"deep":"some note"}},"app":["app"],"msg":" a really long message a really long message a really long message a really long message a really long message a really long message a really long message a really long message a really long message a really long message a really long message a really long message a really long message a really long message a really long message a really long message a really long message a really long message a really long message a really long message a really long message a really long message a really long message a really long message a really long message a really long message a really long message a really long message a really long message a really long message a really long message a really long message a really long message a really long message a really long message a really long message a really long message a really long message a really long message a really long message","pid":"7331","loc":null,"host":"example","sev":"Info","thread":"1337"}
23+{"at":"2016-06-12T12:34:56Z","env":"production","ns":["foo"],"data":{"note":{"deep":"some note"}},"app":["app"],"msg":"сообщение","pid":"7331","loc":null,"host":"example","sev":"Info","thread":"1337"}
24+{"at":"2016-06-12T12:34:56Z","env":"production","ns":["foo"],"data":{"note":{"deep":"some note"}},"app":["app"],"msg":"哈囉世界","pid":"7331","loc":null,"host":"example","sev":"Info","thread":"1337"}
25+{"at":"2000-01-01T00:00:00Z","env":"production","ns":["foo"],"data":{"note":{"deep":"some note"}},"app":["app"],"msg":"message","pid":"7331","loc":null,"host":"example","sev":"Info","thread":"1337"}
26+{"at":"2123-12-31T23:59:59.999999999999Z","env":"production","ns":["foo"],"data":{"note":{"deep":"some note"}},"app":["app"],"msg":"message","pid":"7331","loc":null,"host":"example","sev":"Info","thread":"1337"}
27+{"at":"2016-10-10T01:01:05Z","env":"production","ns":["foo"],"data":{"note":{"deep":"some note"}},"app":["app"],"msg":"message","pid":"7331","loc":null,"host":"example","sev":"Info","thread":"1337"}
28+{"at":"2100-12-31T12:59:10.1Z","env":"production","ns":["foo"],"data":{"note":{"deep":"some note"}},"app":["app"],"msg":"message","pid":"7331","loc":null,"host":"example","sev":"Info","thread":"1337"}
29+{"at":"1982-01-01T12:30:00.000000000001Z","env":"production","ns":["foo"],"data":{"note":{"deep":"some note"}},"app":["app"],"msg":"message","pid":"7331","loc":null,"host":"example","sev":"Info","thread":"1337"}
30+{"at":"2016-06-12T12:34:56Z","env":"production","ns":["foo"],"data":{"note":{"deep":"some note"}},"app":["app"],"msg":"message","pid":"7331","loc":null,"host":"example","sev":"Info","thread":"1337"}
31+{"at":"2016-06-12T12:34:56Z","env":"production","ns":["foo","bar"],"data":{"note":{"deep":"some note"}},"app":["app"],"msg":"message","pid":"7331","loc":null,"host":"example","sev":"Info","thread":"1337"}
32+{"at":"2016-06-12T12:34:56Z","env":"production","ns":["фу","бар"],"data":{"note":{"deep":"some note"}},"app":["app"],"msg":"message","pid":"7331","loc":null,"host":"example","sev":"Info","thread":"1337"}
33+{"at":"2016-06-12T12:34:56Z","env":"production","ns":["with\nnewline"],"data":{"note":{"deep":"some note"}},"app":["app"],"msg":"message","pid":"7331","loc":null,"host":"example","sev":"Info","thread":"1337"}
34+{"at":"2016-06-12T12:34:56Z","env":"production","ns":["foo"],"data":{"note":{"deep":"some note"}},"app":["app"],"msg":"message","pid":"7331","loc":null,"host":"example","sev":"Info","thread":"1337"}
35+{"at":"2016-06-12T12:34:56Z","env":"production","ns":["foo"],"data":{"note":{"deep":"some note"}},"app":["app"],"msg":"message","pid":"7331","loc":{"loc_col":1,"loc_pkg":"main","loc_mod":"Some.Module","loc_fn":"path/Some/Module.hs","loc_ln":30},"host":"example","sev":"Info","thread":"1337"}
36+{"at":"2016-06-12T12:34:56Z","env":"production","ns":["foo"],"data":{"note":{"deep":"some note"}},"app":["app"],"msg":"message","pid":"7331","loc":{"loc_col":9000,"loc_pkg":"main","loc_mod":"Some.Module","loc_fn":"путь/Some/Module.hs","loc_ln":3000},"host":"example","sev":"Info","thread":"1337"}
37+{"at":"2016-06-12T12:34:56Z","env":"production","ns":["foo"],"data":{"sub":null,"list":[],"text":"","num":0,"float":0},"app":["app"],"msg":"message","pid":"7331","loc":null,"host":"example","sev":"Info","thread":"1337"}
38+{"at":"2016-06-12T12:34:56Z","env":"production","ns":["foo"],"data":{"sub":{"note":{"deep":"some note"}},"list":["one","two","three"],"text":"note","num":10,"float":5.5},"app":["app"],"msg":"message","pid":"7331","loc":null,"host":"example","sev":"Info","thread":"1337"}
katip/test/Katip/Tests/Scribes/Handle-text.golden+38−38
@@ -1,40 +1,40 @@
1−[2016-06-12 12:34:56][foo][Debug][example][7331][1337][note.deep:some note] message
2−[2016-06-12 12:34:56][foo][Info][example][7331][1337][note.deep:some note] message
3−[2016-06-12 12:34:56][foo][Notice][example][7331][1337][note.deep:some note] message
4−[2016-06-12 12:34:56][foo][Warning][example][7331][1337][note.deep:some note] message
5−[2016-06-12 12:34:56][foo][Error][example][7331][1337][note.deep:some note] message
6−[2016-06-12 12:34:56][foo][Critical][example][7331][1337][note.deep:some note] message
7−[2016-06-12 12:34:56][foo][Alert][example][7331][1337][note.deep:some note] message
8−[2016-06-12 12:34:56][foo][Emergency][example][7331][1337][note.deep:some note] message
9−[2016-06-12 12:34:56][foo][Info][example][7331][0][note.deep:some note] message
10−[2016-06-12 12:34:56][foo][Info][example][7331][1][note.deep:some note] message
11−[2016-06-12 12:34:56][foo][Info][example][7331][1337][note.deep:some note] message
12−[2016-06-12 12:34:56][foo][Info][example][7331][2147483647][note.deep:some note] message
13−[2016-06-12 12:34:56][foo][Info][example][7331][1337][note.deep:some note] message
14−[2016-06-12 12:34:56][foo][Info][www.example.com][7331][1337][note.deep:some note] message
15−[2016-06-12 12:34:56][foo][Info][127.0.0.1][7331][1337][note.deep:some note] message
16−[2016-06-12 12:34:56][foo][Info][example][0][1337][note.deep:some note] message
17−[2016-06-12 12:34:56][foo][Info][example][1][1337][note.deep:some note] message
18−[2016-06-12 12:34:56][foo][Info][example][1337][1337][note.deep:some note] message
19−[2016-06-12 12:34:56][foo][Info][example][2147483647][1337][note.deep:some note] message
20−[2016-06-12 12:34:56][foo][Info][example][7331][1337][note.deep:some note] message
21−[2016-06-12 12:34:56][foo][Info][example][7331][1337][note.deep:some note] message
1+[2016-06-12 12:34:56][foo][Debug][example][PID 7331][ThreadId 1337][note.deep:some note] message
2+[2016-06-12 12:34:56][foo][Info][example][PID 7331][ThreadId 1337][note.deep:some note] message
3+[2016-06-12 12:34:56][foo][Notice][example][PID 7331][ThreadId 1337][note.deep:some note] message
4+[2016-06-12 12:34:56][foo][Warning][example][PID 7331][ThreadId 1337][note.deep:some note] message
5+[2016-06-12 12:34:56][foo][Error][example][PID 7331][ThreadId 1337][note.deep:some note] message
6+[2016-06-12 12:34:56][foo][Critical][example][PID 7331][ThreadId 1337][note.deep:some note] message
7+[2016-06-12 12:34:56][foo][Alert][example][PID 7331][ThreadId 1337][note.deep:some note] message
8+[2016-06-12 12:34:56][foo][Emergency][example][PID 7331][ThreadId 1337][note.deep:some note] message
9+[2016-06-12 12:34:56][foo][Info][example][PID 7331][ThreadId 0][note.deep:some note] message
10+[2016-06-12 12:34:56][foo][Info][example][PID 7331][ThreadId 1][note.deep:some note] message
11+[2016-06-12 12:34:56][foo][Info][example][PID 7331][ThreadId 1337][note.deep:some note] message
12+[2016-06-12 12:34:56][foo][Info][example][PID 7331][ThreadId 2147483647][note.deep:some note] message
13+[2016-06-12 12:34:56][foo][Info][example][PID 7331][ThreadId 1337][note.deep:some note] message
14+[2016-06-12 12:34:56][foo][Info][www.example.com][PID 7331][ThreadId 1337][note.deep:some note] message
15+[2016-06-12 12:34:56][foo][Info][127.0.0.1][PID 7331][ThreadId 1337][note.deep:some note] message
16+[2016-06-12 12:34:56][foo][Info][example][PID 0][ThreadId 1337][note.deep:some note] message
17+[2016-06-12 12:34:56][foo][Info][example][PID 1][ThreadId 1337][note.deep:some note] message
18+[2016-06-12 12:34:56][foo][Info][example][PID 1337][ThreadId 1337][note.deep:some note] message
19+[2016-06-12 12:34:56][foo][Info][example][PID 2147483647][ThreadId 1337][note.deep:some note] message
20+[2016-06-12 12:34:56][foo][Info][example][PID 7331][ThreadId 1337][note.deep:some note] message
21+[2016-06-12 12:34:56][foo][Info][example][PID 7331][ThreadId 1337][note.deep:some note] message
2222with newline
23−[2016-06-12 12:34:56][foo][Info][example][7331][1337][note.deep:some note] a really long message a really long message a really long message a really long message a really long message a really long message a really long message a really long message a really long message a really long message a really long message a really long message a really long message a really long message a really long message a really long message a really long message a really long message a really long message a really long message a really long message a really long message a really long message a really long message a really long message a really long message a really long message a really long message a really long message a really long message a really long message a really long message a really long message a really long message a really long message a really long message a really long message a really long message a really long message a really long message
24−[2016-06-12 12:34:56][foo][Info][example][7331][1337][note.deep:some note] сообщение
25−[2016-06-12 12:34:56][foo][Info][example][7331][1337][note.deep:some note] 哈囉世界
26−[2000-01-01 00:00:00][foo][Info][example][7331][1337][note.deep:some note] message
27−[2123-12-31 23:59:59][foo][Info][example][7331][1337][note.deep:some note] message
28−[2016-10-10 01:01:05][foo][Info][example][7331][1337][note.deep:some note] message
29−[2100-12-31 12:59:10][foo][Info][example][7331][1337][note.deep:some note] message
30−[1982-01-01 12:30:00][foo][Info][example][7331][1337][note.deep:some note] message
31−[2016-06-12 12:34:56][foo][Info][example][7331][1337][note.deep:some note] message
32−[2016-06-12 12:34:56][foo.bar][Info][example][7331][1337][note.deep:some note] message
33−[2016-06-12 12:34:56][фу.бар][Info][example][7331][1337][note.deep:some note] message
23+[2016-06-12 12:34:56][foo][Info][example][PID 7331][ThreadId 1337][note.deep:some note] a really long message a really long message a really long message a really long message a really long message a really long message a really long message a really long message a really long message a really long message a really long message a really long message a really long message a really long message a really long message a really long message a really long message a really long message a really long message a really long message a really long message a really long message a really long message a really long message a really long message a really long message a really long message a really long message a really long message a really long message a really long message a really long message a really long message a really long message a really long message a really long message a really long message a really long message a really long message a really long message
24+[2016-06-12 12:34:56][foo][Info][example][PID 7331][ThreadId 1337][note.deep:some note] сообщение
25+[2016-06-12 12:34:56][foo][Info][example][PID 7331][ThreadId 1337][note.deep:some note] 哈囉世界
26+[2000-01-01 00:00:00][foo][Info][example][PID 7331][ThreadId 1337][note.deep:some note] message
27+[2123-12-31 23:59:59][foo][Info][example][PID 7331][ThreadId 1337][note.deep:some note] message
28+[2016-10-10 01:01:05][foo][Info][example][PID 7331][ThreadId 1337][note.deep:some note] message
29+[2100-12-31 12:59:10][foo][Info][example][PID 7331][ThreadId 1337][note.deep:some note] message
30+[1982-01-01 12:30:00][foo][Info][example][PID 7331][ThreadId 1337][note.deep:some note] message
31+[2016-06-12 12:34:56][foo][Info][example][PID 7331][ThreadId 1337][note.deep:some note] message
32+[2016-06-12 12:34:56][foo.bar][Info][example][PID 7331][ThreadId 1337][note.deep:some note] message
33+[2016-06-12 12:34:56][фу.бар][Info][example][PID 7331][ThreadId 1337][note.deep:some note] message
3434[2016-06-12 12:34:56][with
35−newline][Info][example][7331][1337][note.deep:some note] message
36−[2016-06-12 12:34:56][foo][Info][example][7331][1337][note.deep:some note] message
37−[2016-06-12 12:34:56][foo][Info][example][7331][1337][note.deep:some note][main:Some.Module path/Some/Module.hs:30:1] message
38−[2016-06-12 12:34:56][foo][Info][example][7331][1337][note.deep:some note][main:Some.Module путь/Some/Module.hs:3000:9000] message
39−[2016-06-12 12:34:56][foo][Info][example][7331][1337][sub:null][text:][num:0.0][float:0.0] message
40−[2016-06-12 12:34:56][foo][Info][example][7331][1337][sub.note.deep:some note][text:note][num:10.0][float:5.5] message
35+newline][Info][example][PID 7331][ThreadId 1337][note.deep:some note] message
36+[2016-06-12 12:34:56][foo][Info][example][PID 7331][ThreadId 1337][note.deep:some note] message
37+[2016-06-12 12:34:56][foo][Info][example][PID 7331][ThreadId 1337][note.deep:some note][main:Some.Module path/Some/Module.hs:30:1] message
38+[2016-06-12 12:34:56][foo][Info][example][PID 7331][ThreadId 1337][note.deep:some note][main:Some.Module путь/Some/Module.hs:3000:9000] message
39+[2016-06-12 12:34:56][foo][Info][example][PID 7331][ThreadId 1337][sub:null][text:][num:0][float:0] message
40+[2016-06-12 12:34:56][foo][Info][example][PID 7331][ThreadId 1337][sub.note.deep:some note][text:note][num:10][float:5.5] message
katip/test/Katip/Tests/Scribes/Handle.hs+42−12
@@ -1,4 +1,5 @@
11{-# LANGUAGE OverloadedStrings #-}
2+{-# LANGUAGE RankNTypes #-}
23module Katip.Tests.Scribes.Handle
34 ( tests
45 ) where
@@ -6,9 +7,9 @@ module Katip.Tests.Scribes.Handle
67-------------------------------------------------------------------------------
78import Control.Monad
89import Data.Aeson
9−import qualified Data.ByteString.Lazy as BL
1010import qualified Data.ByteString.Char8 as B
11−import Data.Monoid
11+import qualified Data.ByteString.Lazy as BL
12+import Data.Monoid as M
1213import Data.Text (Text)
1314import qualified Data.Text as T
1415import qualified Data.Text.Encoding as T
@@ -25,7 +26,6 @@ import Test.Tasty.HUnit
2526import Text.Regex.TDFA
2627-------------------------------------------------------------------------------
2728import Katip
28−import Katip.Scribes.Handle
2929-------------------------------------------------------------------------------
3030
3131
@@ -39,13 +39,26 @@ tests = testGroup "Katip.Scribes.Handle"
3939 runKatipT le $ logItem dummyLogItem "test" Nothing InfoS "wont make it in"
4040 hClose h
4141 res <- readFile path
42− let pat = "\\[[[:digit:]]{4}-[[:digit:]]{2}-[[:digit:]]{2} [[:digit:]]{2}:[[:digit:]]{2}:[[:digit:]]{2}\\]\\[katip-test.test\\]\\[Info\\]\\[.+\\]\\[[[:digit:]]+\\]\\[ThreadId [[:digit:]]+\\]\\[note.deep:some note\\] test message" :: String
42+ let pat = "\\[[[:digit:]]{4}-[[:digit:]]{2}-[[:digit:]]{2} [[:digit:]]{2}:[[:digit:]]{2}:[[:digit:]]{2}\\]\\[katip-test.test\\]\\[Info\\]\\[.+\\]\\[[PID [:digit:]]+\\]\\[ThreadId [[:digit:]]+\\]\\[note.deep:some note\\] test message" :: String
4343 let matches = res =~ pat
44− assertBool (show res <> " did not match") matches
44+ assertBool (show res M.<> " did not match") matches
45+ , withResource setupFile (const (return ())) $ \setupScribe -> testCase "logs correct data to a file" $ do
46+ (path, fin, le) <- setupScribe
47+ runKatipT le $ logItem dummyLogItem "test" Nothing InfoS "test message"
48+ fin
49+ runKatipT le $ logItem dummyLogItem "test" Nothing InfoS "wont make it in"
50+ res <- readFile path
51+ let pat = "\\[[[:digit:]]{4}-[[:digit:]]{2}-[[:digit:]]{2} [[:digit:]]{2}:[[:digit:]]{2}:[[:digit:]]{2}\\]\\[katip-test.test\\]\\[Info\\]\\[.+\\]\\[[PID [:digit:]]+\\]\\[ThreadId [[:digit:]]+\\]\\[note.deep:some note\\] test message" :: String
52+ let matches = res =~ pat
53+ assertBool (show res <> " did not match") matches
54+ , withResource setupTempFile teardownTempFile $ \setupFn ->
55+ goldenVsString "Text-golden"
56+ "test/Katip/Tests/Scribes/Handle-text.golden"
57+ (setupFn >>= writeTextLog)
4558 , withResource setupTempFile teardownTempFile $ \setupFn ->
46− goldenVsString "Text-golden"
47− "test/Katip/Tests/Scribes/Handle-text.golden"
48− (setupFn >>= writeTextLog)
59+ goldenVsString "Json-golden"
60+ "test/Katip/Tests/Scribes/Handle-json.golden"
61+ (setupFn >>= writeJsonLog)
4962 ]
5063
5164
@@ -79,19 +92,30 @@ setup :: IO (FilePath, Handle, IO (), LogEnv)
7992setup = do
8093 tempDir <- getTemporaryDirectory
8194 (fp, h) <- openTempFile tempDir "katip.log"
82− s <- mkHandleScribe (ColorLog False) h DebugS V3
95+ s <- mkHandleScribe (ColorLog False) h (permitItem DebugS) V3
8396 le <- initLogEnv "katip-test" "test"
8497 le' <- registerScribe "handle" s defaultScribeSettings le
8598 return (fp, h, void (closeScribes le'), le')
8699
87100
88101-------------------------------------------------------------------------------
89−teardown :: (FilePath, Handle, IO (), LogEnv) -> IO ()
102+teardown :: (a, Handle, b, c) -> IO ()
90103teardown (_, h, _, _) = do
91104 chk <- hIsOpen h
92105 when chk $ hClose h
93106
94107
108+-------------------------------------------------------------------------------
109+setupFile :: IO (FilePath, IO (), LogEnv)
110+setupFile = do
111+ tempDir <- getTemporaryDirectory
112+ (fp, h) <- openTempFile tempDir "katip.log"
113+ hClose h
114+ s <- mkFileScribe fp (permitItem DebugS) V3
115+ le <- initLogEnv "katip-test" "test"
116+ le' <- registerScribe "handle" s defaultScribeSettings le
117+ return (fp, void (closeScribes le'), le')
118+
95119
96120-- Following code tests Handle scribe output against a golden file.
97121-- This test will fail on non utf8 locales because golden file is in utf-8.
@@ -214,14 +238,20 @@ mkUTCTime y mt d h mn s = UTCTime day dt
214238
215239-------------------------------------------------------------------------------
216240writeTextLog :: (FilePath, Handle) -> IO (BL.ByteString)
217−writeTextLog (path, h) = do
241+writeTextLog = writeFormattedLog bracketFormat
242+
243+writeJsonLog :: (FilePath, Handle) -> IO (BL.ByteString)
244+writeJsonLog = writeFormattedLog jsonFormat
245+
246+writeFormattedLog :: (forall a . LogItem a => ItemFormatter a) -> (FilePath, Handle) -> IO (BL.ByteString)
247+writeFormattedLog format (path, h) = do
218248 mapM_ (put . formatOne) genItems
219249 mapM_ (put . formatOne) genTypedItems
220250 hClose h
221251 BL.readFile path
222252 where
223253 formatOne :: LogItem a => Item a -> Text
224− formatOne = LT.toStrict . LT.toLazyText . formatItem False V3
254+ formatOne = LT.toStrict . LT.toLazyText . format False V3
225255 put = B.hPutStrLn h . T.encodeUtf8
226256
227257setupTempFile :: IO (FilePath, Handle)
stack-lts-2.yamldeleted−32
@@ -1,32 +0,0 @@
1−resolver: lts-2.22
2−packages:
3− - katip
4− - katip-elasticsearch
5−extra-deps:
6− - bloodhound-0.14.0.0
7− - string-conv-0.1.2
8− - aeson-0.11.1.0
9− - http-client-0.5.4
10− - fail-4.9.0.0
11− - tagged-0.8.3
12− - uuid-1.3.12
13− - attoparsec-0.13.1.0
14− - retry-0.7.4.2
15− - tasty-0.11.0.4
16− - time-locale-compat-0.1.1.1
17− - cryptonite-0.20
18− - microlens-0.4.7.0
19− - microlens-th-0.4.1.0
20− - time-1.5.0.1
21− - memory-0.13
22− - rng-utils-0.2.1
23− - lens-aeson-1.0.0.5
24− - atomic-primops-0.8.0.4
25− - safe-exceptions-0.1.5.0
26−flags:
27− katip:
28− lib-Werror: true
29− katip-elasticsearch:
30− lib-Werror: true
31− time-locale-compat:
32− old-locale: false
stack-lts-3.yamldeleted−22
@@ -1,22 +0,0 @@
1−resolver: lts-3.22
2−packages:
3− - katip
4− - katip-elasticsearch
5−extra-deps:
6− - bloodhound-0.14.0.0
7− - string-conv-0.1.2
8− - aeson-0.11.1.0
9− - http-client-0.5.4
10− - fail-4.9.0.0
11− - tagged-0.8.3
12− - uuid-1.3.12
13− - attoparsec-0.13.1.0
14− - retry-0.7.4.2
15− - tasty-0.11.0.4
16− - rng-utils-0.2.1
17− - safe-exceptions-0.1.5.0
18−flags:
19− katip:
20− lib-Werror: true
21− katip-elasticsearch:
22− lib-Werror: true
stack-lts-4.yamldeleted−19
@@ -1,19 +0,0 @@
1−resolver: lts-4.2
2−packages:
3− - katip
4− - katip-elasticsearch
5−extra-deps:
6− - bloodhound-0.14.0.0
7− - string-conv-0.1.2
8− - aeson-0.11.1.0
9− - http-client-0.5.4
10− - fail-4.9.0.0
11− - tagged-0.8.3
12− - uuid-1.3.12
13− - rng-utils-0.2.1
14− - safe-exceptions-0.1.5.0
15−flags:
16− katip:
17− lib-Werror: true
18− katip-elasticsearch:
19− lib-Werror: true
stack-lts-5.yamldeleted−16
@@ -1,16 +0,0 @@
1−resolver: lts-5.18
2−packages:
3− - katip
4− - katip-elasticsearch
5−extra-deps:
6− - bloodhound-0.14.0.0
7− - string-conv-0.1.2
8− - aeson-0.11.1.0
9− - http-client-0.5.4
10− - rng-utils-0.2.1
11− - safe-exceptions-0.1.5.0
12−flags:
13− katip:
14− lib-Werror: true
15− katip-elasticsearch:
16− lib-Werror: true
stack.yaml+11−2
@@ -1,11 +1,20 @@
1−resolver: lts-8.21
1+resolver: lts-16.18
22packages:
33 - katip
44 - katip-elasticsearch
5+ - katip-datadog
6+ - katip-logzio
57extra-deps:
6− - bloodhound-0.14.0.0
8+ # ghc 8.8 support
9+ - github: wireapp/bloodhound
10+ commit: b409f52b518f20a4c20651b5893c31b7b85cb9b1
11+allow-newer: true
712flags:
813 katip:
914 lib-Werror: true
1015 katip-elasticsearch:
1116 lib-Werror: true
17+ katip-datadog:
18+ lib-Werror: true
19+ katip-logzio:
20+ lib-Werror: true