@@ -1,53 +1,109 @@
1+{-# LANGUAGE CPP #-}
2+{-# LANGUAGE DefaultSignatures #-}
3+{-# LANGUAGE DeriveFunctor #-}
14{-# LANGUAGE DeriveGeneric #-}
5+{-# LANGUAGE ExistentialQuantification #-}
26{-# LANGUAGE FlexibleContexts #-}
37{-# LANGUAGE FlexibleInstances #-}
8+{-# LANGUAGE GADTs #-}
49{-# LANGUAGE GeneralizedNewtypeDeriving #-}
510{-# LANGUAGE MultiParamTypeClasses #-}
611{-# LANGUAGE OverloadedStrings #-}
712{-# LANGUAGE RankNTypes #-}
813{-# LANGUAGE RecordWildCards #-}
914{-# LANGUAGE TemplateHaskell #-}
10−
15+{-# LANGUAGE TypeFamilies #-}
16+{-# LANGUAGE UndecidableInstances #-}
17+#if MIN_VERSION_base(4, 9, 0)
18+{-# OPTIONS_GHC -fno-warn-redundant-constraints #-}
19+#endif
20+-- | This module is not meant to be imported directly and may contain
21+-- internal mechanisms that will change without notice.
1122module Katip.Core where
1223
1324-------------------------------------------------------------------------------
14−import Control.Applicative
25+import Control.Applicative as A
1526import Control.AutoUpdate
1627import Control.Concurrent
17−import Control.Lens
18−import Control.Monad
19−import Control.Monad.Catch
28+import qualified Control.Concurrent.Async as Async
29+import Control.Concurrent.STM
30+import qualified Control.Concurrent.STM.TBQueue as BQ
31+import Control.Exception.Safe
32+import Control.Monad (unless, void, when)
33+import Control.Monad.Base
34+#if MIN_VERSION_base(4, 9, 0)
35+import qualified Control.Monad.Fail as MF
36+#endif
2037import Control.Monad.IO.Class
38+import Control.Monad.IO.Unlift
2139import Control.Monad.Trans.Class
40+import Control.Monad.Trans.Control
41+#if !MIN_VERSION_either(4, 5, 0)
2242import Control.Monad.Trans.Either
43+#endif
44+import Control.Monad.Trans.Except
2345import Control.Monad.Trans.Maybe
2446import Control.Monad.Trans.Reader
25−import Control.Monad.Trans.State
26−import Control.Monad.Trans.Writer
27−import Data.Aeson (ToJSON (..))
28−import qualified Data.Aeson as A
29−import Data.Aeson.Lens
30−import qualified Data.HashMap.Strict as HM
47+import Control.Monad.Trans.Resource (MonadResource, ResourceT,
48+ transResourceT)
49+import Control.Monad.Trans.RWS.Lazy (RWST, mapRWST)
50+import qualified Control.Monad.Trans.RWS.Strict as Strict (RWST, mapRWST)
51+import Control.Monad.Trans.State.Lazy (StateT, mapStateT)
52+import qualified Control.Monad.Trans.State.Strict as Strict (StateT, mapStateT)
53+import Control.Monad.Trans.Writer.Lazy (WriterT, mapWriterT)
54+import qualified Control.Monad.Trans.Writer.Strict as Strict (WriterT,
55+ mapWriterT)
56+import Data.Aeson (FromJSON (..), ToJSON (..),
57+ object)
58+import qualified Data.Aeson as A
59+import Data.Foldable as FT
60+import qualified Data.HashMap.Strict as HM
3161import Data.List
32−import qualified Data.Map.Strict as M
33−import Data.Monoid
62+import qualified Data.Map.Strict as M
63+import Data.Maybe (fromMaybe)
64+import Data.Semigroup as SG
3465import Data.String
3566import Data.String.Conv
36−import Data.Text (Text)
37−import qualified Data.Text as T
38−import qualified Data.Text.Lazy.Builder as B
67+import Data.Text (Text)
68+import qualified Data.Text as T
69+import qualified Data.Text.Lazy.Builder as B
3970import Data.Time
40−import GHC.Generics hiding (to)
71+import GHC.Generics hiding (to)
72+#if MIN_VERSION_base(4, 8, 0)
73+#if !MIN_VERSION_base(4, 9, 0)
74+import GHC.SrcLoc
75+#endif
76+import GHC.Stack
77+#endif
4178import Language.Haskell.TH
42−import qualified Language.Haskell.TH.Syntax as TH
79+import qualified Language.Haskell.TH.Syntax as TH
80+import Lens.Micro
81+import Lens.Micro.TH
4382import Network.HostName
83+#if mingw32_HOST_OS
84+import Katip.Compat
85+#else
4486import System.Posix
87+#endif
88+
4589-------------------------------------------------------------------------------
4690
4791
92+readMay :: Read a => String -> Maybe a
93+readMay s = case [x | (x,t) <- reads s, ("","") <- lex t] of
94+ [x] -> Just x
95+ [] -> Nothing -- no parse
96+ _ -> Nothing -- Ambiguous parse
97+
98+
4899-------------------------------------------------------------------------------
49−newtype Namespace = Namespace { getNamespace :: [Text] }
50− deriving (Eq,Show,Read,Ord,Generic,ToJSON,Monoid)
100+-- | Represents a heirarchy of namespaces going from general to
101+-- specific. For instance: ["processname", "subsystem"]. Note that
102+-- single-segment namespaces can be created using
103+-- IsString/OverloadedStrings, so "foo" will result in Namespace
104+-- ["foo"].
105+newtype Namespace = Namespace { unNamespace :: [Text] }
106+ deriving (Eq,Show,Read,Ord,Generic,ToJSON,FromJSON,SG.Semigroup,Monoid)
51107
52108instance IsString Namespace where
53109 fromString s = Namespace [fromString s]
@@ -62,7 +118,7 @@ intercalateNs (Namespace xs) = intersperse "." xs
62118-------------------------------------------------------------------------------
63119-- | Application environment, like @prod@, @devel@, @testing@.
64120newtype Environment = Environment { getEnvironment :: Text }
65− deriving (Eq,Show,Read,Ord,Generic,ToJSON,IsString)
121+ deriving (Eq,Show,Read,Ord,Generic,ToJSON,FromJSON,IsString)
66122
67123
68124-------------------------------------------------------------------------------
@@ -75,7 +131,7 @@ data Severity
75131 | CriticalS -- ^ Severe situations
76132 | AlertS -- ^ Take immediate action
77133 | EmergencyS -- ^ System is unusable
78− deriving (Eq, Ord, Show, Read, Generic)
134+ deriving (Eq, Ord, Show, Read, Generic, Enum, Bounded)
79135
80136
81137-------------------------------------------------------------------------------
@@ -87,89 +143,206 @@ data Severity
87143-- - 'V3' implies the maximum amount of payload information.
88144-- - Anything in between is left to the discretion of the developer.
89145data Verbosity = V0 | V1 | V2 | V3
90− deriving (Eq, Ord, Show, Read, Generic)
146+ deriving (Eq, Ord, Show, Read, Generic, Enum, Bounded)
91147
92148
93149-------------------------------------------------------------------------------
94150renderSeverity :: Severity -> Text
95151renderSeverity s = case s of
96− DebugS -> "Debug"
97− InfoS -> "Info"
98− NoticeS -> "Notice"
99− WarningS -> "Warning"
100− ErrorS -> "Error"
101− CriticalS -> "Critical"
102− AlertS -> "Alert"
152+ DebugS -> "Debug"
153+ InfoS -> "Info"
154+ NoticeS -> "Notice"
155+ WarningS -> "Warning"
156+ ErrorS -> "Error"
157+ CriticalS -> "Critical"
158+ AlertS -> "Alert"
103159 EmergencyS -> "Emergency"
104160
105161
106162-------------------------------------------------------------------------------
163+textToSeverity :: Text -> Maybe Severity
164+textToSeverity = go . T.toLower
165+ where
166+ go "debug" = Just DebugS
167+ go "info" = Just InfoS
168+ go "notice" = Just NoticeS
169+ go "warning" = Just WarningS
170+ go "error" = Just ErrorS
171+ go "critical" = Just CriticalS
172+ go "alert" = Just AlertS
173+ go "emergency" = Just EmergencyS
174+ go _ = Nothing
175+
176+
107177instance ToJSON Severity where
108178 toJSON s = A.String (renderSeverity s)
109179
180+instance FromJSON Severity where
181+ parseJSON = A.withText "Severity" parseSeverity
182+ where
183+ parseSeverity t = case textToSeverity t of
184+ Just x -> return x
185+ Nothing -> fail $ "Invalid Severity " ++ toS t
186+
187+instance ToJSON Verbosity where
188+ toJSON s = A.String $ case s of
189+ V0 -> "V0"
190+ V1 -> "V1"
191+ V2 -> "V2"
192+ V3 -> "V3"
193+
194+instance FromJSON Verbosity where
195+ parseJSON = A.withText "Verbosity" $ \s -> case s of
196+ "V0" -> return V0
197+ "V1" -> return V1
198+ "V2" -> return V2
199+ "V3" -> return V3
200+ _ -> fail $ "Invalid Verbosity " ++ toS s
201+
110202
111203-------------------------------------------------------------------------------
112−-- | Log message with Builder unerneath; use '<>' to concat in O(1).
204+-- | Log message with Builder underneath; use '<>' to concat in O(1).
113205newtype LogStr = LogStr { unLogStr :: B.Builder }
114− deriving (Generic)
206+ deriving (Generic, Show, Eq)
115207
116208instance IsString LogStr where
117209 fromString = LogStr . B.fromString
118210
211+
212+instance Semigroup LogStr where
213+ (LogStr a) <> (LogStr b) = LogStr (a <> b)
214+
215+
119216instance Monoid LogStr where
120− mappend (LogStr a) (LogStr b) = LogStr (a `mappend` b)
217+ mappend = (<>)
121218 mempty = LogStr mempty
122219
220+
221+instance FromJSON LogStr where
222+ parseJSON = A.withText "LogStr" parseLogStr
223+ where
224+ parseLogStr = return . LogStr . B.fromText
225+
123226-------------------------------------------------------------------------------
124−-- | Pack any string-like thing into a 'LogMsg'. This will
125−-- automatically work on 'String', 'ByteString, 'Text' and any of the
227+-- | Pack any string-like thing into a 'LogStr'. This will
228+-- automatically work on 'String', 'ByteString', 'Text' and any of the
126229-- lazy variants.
127230logStr :: StringConv a Text => a -> LogStr
128231logStr t = LogStr (B.fromText $ toS t)
129232
130233
131234-------------------------------------------------------------------------------
132−-- | Shorthand for 'logMsg'
235+-- | Shorthand for 'logStr'
133236ls :: StringConv a Text => a -> LogStr
134237ls = logStr
135238
136239
137240-------------------------------------------------------------------------------
241+-- | Convert any showable type into a 'LogStr'.
138242showLS :: Show a => a -> LogStr
139243showLS = ls . show
140244
141245
246+-------------------------------------------------------------------------------
247+newtype ThreadIdText = ThreadIdText {
248+ getThreadIdText :: Text
249+ } deriving (ToJSON, FromJSON, Show, Eq, Ord)
250+
251+
252+mkThreadIdText :: ThreadId -> ThreadIdText
253+mkThreadIdText = ThreadIdText . stripPrefix' "ThreadId " . T.pack . show
254+ where
255+ stripPrefix' pfx t = fromMaybe t (T.stripPrefix pfx t)
256+
257+
142258-------------------------------------------------------------------------------
143259-- | This has everything each log message will contain.
144260data Item a = Item {
145− itemApp :: Namespace
146− , itemEnv :: Environment
147− , itemSeverity :: Severity
148− , itemThread :: ThreadId
149− , itemHost :: HostName
150− , itemProcess :: ProcessID
151− , itemPayload :: a
152− , itemMessage :: LogStr
153− , itemTime :: UTCTime
154− , itemNamespace :: Namespace
155− , itemLoc :: Maybe Loc
156− } deriving (Generic)
157−
261+ _itemApp :: Namespace
262+ , _itemEnv :: Environment
263+ , _itemSeverity :: Severity
264+ , _itemThread :: ThreadIdText
265+ , _itemHost :: HostName
266+ , _itemProcess :: ProcessID
267+ , _itemPayload :: a
268+ , _itemMessage :: LogStr
269+ , _itemTime :: UTCTime
270+ , _itemNamespace :: Namespace
271+ , _itemLoc :: Maybe Loc
272+ } deriving (Generic, Functor)
273+makeLenses ''Item
274+
275+-- Manual instance because 'Loc' has no 'Eq' and 'Show' instances in old
276+-- versions of template-haskell (< 2.10)
277+instance Eq a => Eq (Item a) where
278+ a == b = FT.and [ _itemApp a == _itemApp b
279+ , _itemEnv a == _itemEnv b
280+ , _itemSeverity a == _itemSeverity b
281+ , _itemThread a == _itemThread b
282+ , _itemHost a == _itemHost b
283+ , _itemProcess a == _itemProcess b
284+ , _itemPayload a == _itemPayload b
285+ , _itemMessage a == _itemMessage b
286+ , _itemTime a == _itemTime b
287+ , _itemNamespace a == _itemNamespace b
288+ , case (_itemLoc a, _itemLoc b) of
289+ (Nothing, Nothing) -> True
290+ (Just l1, Just l2) -> FT.and [ loc_filename l1 == loc_filename l2
291+ , loc_package l1 == loc_package l2
292+ , loc_module l1 == loc_module l2
293+ , loc_start l1 == loc_start l2
294+ , loc_end l1 == loc_end l2
295+ ]
296+ _ -> False
297+ ]
298+
299+instance Show a => Show (Item a) where
300+ showsPrec d Item{..} = showParen (d >= 11) ( showString "Item {"
301+ . field "_itemApp" _itemApp
302+ . field "_itemEnv" _itemEnv
303+ . field "_itemSeverity" _itemSeverity
304+ . field "_itemThread" _itemThread
305+ . field "_itemHost" _itemHost
306+ . field "_itemProcess" _itemProcess
307+ . field "_itemPayload" _itemPayload
308+ . field "_itemMessage" _itemMessage
309+ . field "_itemTime" _itemTime
310+ . field "_itemNamespace" _itemNamespace
311+ . showString "_itemLoc = " . shows (LocShow <$> _itemLoc)
312+ . showChar '}'
313+ )
314+ where
315+ field n v = showString n . showString " = " . shows v . showString ", "
316+
317+newtype LocShow = LocShow Loc
318+
319+
320+instance Show LocShow where
321+ showsPrec d (LocShow Loc{..}) = showParen (d >= 11) ( showString "Loc {"
322+ . field "loc_filename" loc_filename
323+ . field "loc_package" loc_package
324+ . field "loc_module" loc_module
325+ . field "loc_start" loc_start
326+ . showString "loc_end = " . shows loc_end
327+ . showChar '}'
328+ )
329+ where
330+ field n v = showString n . showString " = " . shows v . showString ", "
158331
159332
160333instance ToJSON a => ToJSON (Item a) where
161334 toJSON Item{..} = A.object
162− [ "app" A..= itemApp
163− , "env" A..= itemEnv
164− , "sev" A..= itemSeverity
165− , "thread" A..= show itemThread
166− , "host" A..= itemHost
167− , "pid" A..= A.String (toS (show itemProcess))
168− , "data" A..= itemPayload
169− , "msg" A..= (B.toLazyText $ unLogStr itemMessage)
170− , "at" A..= itemTime
171− , "ns" A..= itemNamespace
172− , "loc" A..= fmap LocJs itemLoc
335+ [ "app" A..= _itemApp
336+ , "env" A..= _itemEnv
337+ , "sev" A..= _itemSeverity
338+ , "thread" A..= getThreadIdText _itemThread
339+ , "host" A..= _itemHost
340+ , "pid" A..= ProcessIDJs _itemProcess
341+ , "data" A..= _itemPayload
342+ , "msg" A..= (B.toLazyText $ unLogStr _itemMessage)
343+ , "at" A..= _itemTime
344+ , "ns" A..= _itemNamespace
345+ , "loc" A..= fmap LocJs _itemLoc
173346 ]
174347
175348newtype LocJs = LocJs { getLocJs :: Loc }
@@ -185,59 +358,317 @@ instance ToJSON LocJs where
185358 ]
186359
187360
361+instance FromJSON LocJs where
362+ parseJSON = A.withObject "LocJs" parseLocJs
363+ where
364+ parseLocJs o = do
365+ fn <- o A..: "loc_fn"
366+ p <- o A..: "loc_pkg"
367+ m <- o A..: "loc_mod"
368+ l <- o A..: "loc_ln"
369+ c <- o A..: "loc_col"
370+ return $ LocJs $ Loc fn p m (l, c) (l, c)
371+
372+
373+instance FromJSON a => FromJSON (Item a) where
374+ parseJSON = A.withObject "Item" parseItem
375+ where
376+ parseItem o = Item
377+ <$> o A..: "app"
378+ <*> o A..: "env"
379+ <*> o A..: "sev"
380+ <*> o A..: "thread"
381+ <*> o A..: "host"
382+ <*> (getProcessIDJs <$> o A..: "pid")
383+ <*> o A..: "data"
384+ <*> o A..: "msg"
385+ <*> o A..: "at"
386+ <*> o A..: "ns"
387+ <*> (fmap getLocJs <$> o A..: "loc")
388+
389+
390+processIDToText :: ProcessID -> Text
391+processIDToText = toS . show
392+
393+
394+textToProcessID :: Text -> Maybe ProcessID
395+textToProcessID = readMay . toS
396+
397+
398+newtype ProcessIDJs = ProcessIDJs {
399+ getProcessIDJs :: ProcessID
400+ }
401+
402+
403+instance ToJSON ProcessIDJs where
404+ toJSON (ProcessIDJs p) = A.String (processIDToText p)
405+
406+
407+instance FromJSON ProcessIDJs where
408+ parseJSON = A.withText "ProcessID" parseProcessID
409+ where
410+ parseProcessID t = case textToProcessID t of
411+ Just p -> return $ ProcessIDJs p
412+ Nothing -> fail $ "Invalid ProcessIDJs " ++ toS t
413+
414+
188415-------------------------------------------------------------------------------
189416-- | Field selector by verbosity within JSON payload.
190417data PayloadSelection
191418 = AllKeys
192419 | SomeKeys [Text]
420+ deriving (Show, Eq)
421+
422+instance Semigroup PayloadSelection where
423+ AllKeys <> _ = AllKeys
424+ _ <> AllKeys = AllKeys
425+ SomeKeys as <> SomeKeys bs = SomeKeys (as <> bs)
426+
427+
428+instance Monoid PayloadSelection where
429+ mempty = SomeKeys []
430+ mappend = (<>)
431+
432+
433+-------------------------------------------------------------------------------
434+-- | Katip requires JSON objects to be logged as context. This
435+-- typeclass provides a default instance which uses ToJSON and
436+-- produces an empty object if 'toJSON' results in any type other than
437+-- object. If you have a type you want to log that produces an Array
438+-- or Number for example, you'll want to write an explicit instance
439+-- here. You can trivially add a ToObject instance for something with
440+-- a ToJSON instance like:
441+--
442+-- > instance ToObject Foo
443+class ToObject a where
444+ toObject :: a -> A.Object
445+ default toObject :: ToJSON a => a -> A.Object
446+ toObject v = case toJSON v of
447+ A.Object o -> o
448+ _ -> mempty
449+
450+instance ToObject ()
451+instance ToObject A.Object
193452
194453-------------------------------------------------------------------------------
195−-- | Payload objects need instances of this class.
196−class ToJSON a => LogContext a where
454+-- | Payload objects need instances of this class. LogItem makes it so
455+-- that you can have very verbose items getting logged with lots of
456+-- extra fields but under normal circumstances, if your scribe is
457+-- configured for a lower verbosity level, it will only log a
458+-- selection of those keys. Furthermore, each 'Scribe' can be
459+-- configured with a different 'Verbosity' level. You could even use
460+-- 'registerScribe', 'unregisterScribe', and 'clearScribes' to at
461+-- runtime swap out your existing scribes for more verbose debugging
462+-- scribes if you wanted to.
463+--
464+-- When defining 'payloadKeys', don't redundantly declare the same
465+-- keys for higher levels of verbosity. Each level of verbosity
466+-- automatically and recursively contains all keys from the level
467+-- before it.
468+class ToObject a => LogItem a where
197469
198470 -- | List of keys in the JSON object that should be included in message.
199471 payloadKeys :: Verbosity -> a -> PayloadSelection
200472
201473
202−instance LogContext () where payloadKeys _ _ = SomeKeys []
474+instance LogItem () where payloadKeys _ _ = SomeKeys []
475+
476+
477+data AnyLogPayload = forall a. ToJSON a => AnyLogPayload a
478+
479+newtype SimpleLogPayload = SimpleLogPayload {
480+ unSimpleLogPayload :: [(Text, AnyLogPayload)]
481+ }
482+
483+-------------------------------------------------------------------------------
484+-- | A built-in convenience log payload that won't log anything on 'V0',
485+-- but will log everything in any other level of verbosity. Intended
486+-- for easy in-line usage without having to define new log types.
487+--
488+-- Construct using 'sl' and combine multiple tuples using '<>' from
489+-- 'Monoid'.
490+instance ToJSON SimpleLogPayload where
491+ toJSON (SimpleLogPayload as) = object $ map go as
492+ where go (k, AnyLogPayload v) = k A..= v
493+
494+
495+instance ToObject SimpleLogPayload
496+
497+
498+instance LogItem SimpleLogPayload where
499+ payloadKeys V0 _ = SomeKeys []
500+ payloadKeys _ _ = AllKeys
501+
502+
503+instance Semigroup SimpleLogPayload where
504+ SimpleLogPayload a <> SimpleLogPayload b = SimpleLogPayload (a <> b)
505+
506+
507+instance Monoid SimpleLogPayload where
508+ mempty = SimpleLogPayload []
509+ mappend = (<>)
510+
511+
512+-------------------------------------------------------------------------------
513+-- | Construct a simple log from any JSON item.
514+sl :: ToJSON a => Text -> a -> SimpleLogPayload
515+sl a b = SimpleLogPayload [(a, AnyLogPayload b)]
516+
517+
518+-------------------------------------------------------------------------------
519+-- | Constrain payload based on verbosity. Backends should use this to
520+-- automatically bubble higher verbosity levels to lower ones.
521+payloadObject :: LogItem a => Verbosity -> a -> A.Object
522+payloadObject verb a = case FT.foldMap (flip payloadKeys a) [(V0)..verb] of
523+ AllKeys -> toObject a
524+ SomeKeys ks -> HM.filterWithKey (\ k _ -> k `FT.elem` ks) $ toObject a
203525
204526
205527-------------------------------------------------------------------------------
206−-- | Constrain payload based on verbosity. To be used by backends.
207−payloadJson :: LogContext a => Verbosity -> a -> A.Value
208−payloadJson verb a = case payloadKeys verb a of
209− AllKeys -> toJSON a
210− SomeKeys ks -> toJSON a
211− & _Object %~ HM.filterWithKey (\ k _ -> k `elem` ks)
528+-- | Convert log item to its JSON representation while trimming its
529+-- payload based on the desired verbosity. Backends that push JSON
530+-- messages should use this to obtain their payload.
531+itemJson :: LogItem a => Verbosity -> Item a -> A.Value
532+itemJson verb a = toJSON $ a & itemPayload %~ payloadObject verb
212533
213534
214535-------------------------------------------------------------------------------
215536-- | Scribes are handlers of incoming items. Each registered scribe
216537-- knows how to push a log item somewhere.
538+--
539+-- = Guidelines for writing your own 'Scribe'
540+--
541+-- Scribes should always take a 'Severity' and 'Verbosity'.
542+--
543+-- Severity is used to __exclude log messages__ that are lower than
544+-- the provided Severity. For instance, if the user passes InfoS,
545+-- DebugS items should be ignored. Katip provides the 'permitItem'
546+-- utility for this. The user or the scribe may use 'permitAND' and
547+-- 'permitOR' to further customize this filtering, even dynamically if
548+-- they wish to.
549+--
550+-- Verbosity is used to select keys from the log item's payload. Each
551+-- 'LogItem' instance describes what keys should be retained for each
552+-- Verbosity level. Use the 'payloadObject' utility for extracting the keys
553+-- that should be written.
554+--
555+-- Scribes provide a finalizer IO action ('scribeFinalizer') that is
556+-- meant to synchronously flush any remaining writes and clean up any
557+-- resources acquired when the scribe was created. Internally, katip
558+-- keeps a buffer for each scribe's writes. When 'closeScribe' or
559+-- 'closeScribes' is called, that buffer stops accepting new log
560+-- messages and after the last item in its buffer is sent to 'liPush',
561+-- calls the finalizer. Thus, when the finalizer returns, katip can
562+-- assume that all resources are cleaned up and all log messages are
563+-- durably written.
564+--
565+-- While katip internally buffers messages per 'ScribeSettings', it
566+-- sends them one at a time to the scribe. Depending on the scribe
567+-- itself, it may make sense for that scribe to keep its own internal
568+-- buffer to batch-send logs if writing items one at a time is not
569+-- efficient. The scribe implementer must be sure that on
570+-- finalization, all writes are committed synchronously.
571+
572+-- | Signature of a function passed to `Scribe` constructor and
573+-- mkScribe* functions that decides which messages to be
574+-- logged. Typically filters based on 'Severity', but can be
575+-- combined with other, custom logic with 'permitAND' and 'permitOR'
576+type PermitFunc = forall a. Item a -> IO Bool
577+
578+
579+-- | AND together 2 permit functions
580+permitAND :: PermitFunc -> PermitFunc -> PermitFunc
581+permitAND f1 f2 = \a -> liftA2 (&&) (f1 a) (f2 a)
582+
583+-- | OR together 2 permit functions
584+permitOR :: PermitFunc -> PermitFunc -> PermitFunc
585+permitOR f1 f2 = \a -> liftA2 (||) (f1 a) (f2 a)
586+
587+
217588data Scribe = Scribe {
218− lhPush :: forall a. LogContext a => Item a -> IO ()
219− }
589+ liPush :: forall a. LogItem a => Item a -> IO ()
590+ -- ^ How do we write an item to the scribe's output?
591+ , scribeFinalizer :: IO ()
592+ -- ^ Provide a __blocking__ finalizer to call when your scribe is
593+ -- removed. All pending writes should be flushed synchronously. If
594+ -- this is not relevant to your scribe, return () is fine.
595+ , scribePermitItem :: PermitFunc
596+ -- ^ Provide a filtering function to allow the item to be logged,
597+ -- or not. It can check Severity or some string in item's
598+ -- body. The initial value of this is usually created from
599+ -- 'permitItem'. Scribes and users can customize this by ANDing
600+ -- or ORing onto the default with 'permitAND' or 'permitOR'
601+ }
602+
603+
604+whenM :: Monad m => m Bool -> m () -> m ()
605+whenM mbool = (>>=) mbool . flip when
606+
607+
608+-- | Combine two scribes. Publishes to the left scribe if the left
609+-- would permit the item and to the right scribe if the right would
610+-- permit the item. Finalizers are called in sequence from left to
611+-- right.
612+instance Semigroup Scribe where
613+ (Scribe pushA finA permitA) <> (Scribe pushB finB permitB) =
614+ Scribe (\item -> whenM (permitA item) (pushA item)
615+ >> whenM (permitB item) (pushB item)
616+ )
617+ (finA `finally` finB)
618+ (permitOR permitA permitB)
220619
221620
222621instance Monoid Scribe where
223− mempty = Scribe $ const $ return ()
224− mappend (Scribe a) (Scribe b) = Scribe $ \ item -> do
225− a item
226− b item
622+ mempty = Scribe (const (return ())) (return ()) (permitItem DebugS)
623+ mappend = (<>)
624+
625+
626+-------------------------------------------------------------------------------
627+data ScribeHandle = ScribeHandle {
628+ shScribe :: Scribe
629+ , shChan :: BQ.TBQueue WorkerMessage
630+ }
631+
632+
633+-------------------------------------------------------------------------------
634+data WorkerMessage where
635+ NewItem :: LogItem a => Item a -> WorkerMessage
636+ PoisonPill :: WorkerMessage
637+
638+
639+-------------------------------------------------------------------------------
640+-- | Should this item be logged given the user's maximum severity?
641+-- Most new scribes will use this as a base for their 'PermitFunc'
642+permitItem :: Monad m => Severity -> Item a -> m Bool
643+permitItem sev item = return (_itemSeverity item >= sev)
644+
227645
228646-------------------------------------------------------------------------------
229647data LogEnv = LogEnv {
230648 _logEnvHost :: HostName
231649 , _logEnvPid :: ProcessID
232− , _logEnvNs :: Namespace
650+ , _logEnvApp :: Namespace
651+ -- ^ Name of application. This will typically never change. This
652+ -- field gets prepended to the namespace of your individual log
653+ -- messages. For example, if your app is MyApp and you write a log
654+ -- using "logItem" and the namespace "WebServer", the final
655+ -- namespace will be "MyApp.WebServer"
233656 , _logEnvEnv :: Environment
234657 , _logEnvTimer :: IO UTCTime
235− , _logEnvScribes :: M.Map Text Scribe
658+ -- ^ Action to fetch the timestamp. You can use something like
659+ -- 'AutoUpdate' for high volume logs but note that this may cause
660+ -- some output forms to display logs out of order. Alternatively,
661+ -- you could just use 'getCurrentTime'.
662+ , _logEnvScribes :: M.Map Text ScribeHandle
236663 }
237664makeLenses ''LogEnv
238665
239666
240667-------------------------------------------------------------------------------
668+-- | Create a reasonable default InitLogEnv. Uses an 'AutoUpdate' which
669+-- updates the timer every 1ms. If you need even more timestamp
670+-- precision at the cost of performance, consider setting
671+-- '_logEnvTimer' with 'getCurrentTime'.
241672initLogEnv
242673 :: Namespace
243674 -- ^ A base namespace for this application
@@ -249,66 +680,243 @@ initLogEnv an env = LogEnv
249680 <*> getProcessID
250681 <*> pure an
251682 <*> pure env
252− <*> mkAutoUpdate defaultUpdateSettings { updateAction = getCurrentTime }
683+ <*> mkAutoUpdate defaultUpdateSettings { updateAction = getCurrentTime, updateFreq = 1000 }
253684 <*> pure mempty
254685
255686
256687-------------------------------------------------------------------------------
688+-- | Add a scribe to the list. All future log calls will go to this
689+-- scribe in addition to the others. Writes will be buffered per the
690+-- ScribeSettings to prevent slow scribes from slowing down
691+-- logging. Writes will be dropped if the buffer fills.
257692registerScribe
258693 :: Text
259694 -- ^ Name the scribe
260695 -> Scribe
696+ -> ScribeSettings
261697 -> LogEnv
262− -> LogEnv
263−registerScribe nm h = logEnvScribes . at nm .~ Just h
698+ -> IO LogEnv
699+registerScribe nm scribe ScribeSettings {..} le = do
700+ queue <- atomically (BQ.newTBQueue (fromIntegral _scribeBufferSize))
701+ worker <- spawnScribeWorker scribe queue
702+ let fin = do
703+ atomically (BQ.writeTBQueue queue PoisonPill)
704+ -- wait for our worker to finish final write
705+ void (Async.waitCatch worker)
706+ -- wait for scribe to finish final write
707+ void (scribeFinalizer scribe)
708+
709+ let sh = ScribeHandle (scribe { scribeFinalizer = fin }) queue
710+ return (le & logEnvScribes %~ M.insert nm sh)
711+
712+
713+-------------------------------------------------------------------------------
714+spawnScribeWorker :: Scribe -> BQ.TBQueue WorkerMessage -> IO (Async.Async ())
715+spawnScribeWorker (Scribe write _ _) queue = Async.async go
716+ where
717+ go = do
718+ newCmd <- atomically (BQ.readTBQueue queue)
719+ case newCmd of
720+ NewItem a -> do
721+ -- Swallow any direct exceptions from the
722+ -- scribe. safe-exceptions won't catch async exceptions.
723+ void (tryAny (write a))
724+ go
725+ PoisonPill -> return ()
726+
727+
728+-------------------------------------------------------------------------------
729+data ScribeSettings = ScribeSettings {
730+ _scribeBufferSize :: Int
731+ }
732+ deriving (Show, Eq)
733+
734+makeLenses ''ScribeSettings
735+
736+
737+-- | Reasonable defaults for a scribe. Buffer
738+-- size of 4096.
739+defaultScribeSettings :: ScribeSettings
740+defaultScribeSettings = ScribeSettings 4096
264741
265742
266743-------------------------------------------------------------------------------
744+-- | Remove a scribe from the environment. This does __not__ finalize
745+-- the scribe. This mainly only makes sense to use with something like
746+-- MonadReader's @local@ function to temporarily disavow a single
747+-- logger for a block of code.
267748unregisterScribe
268749 :: Text
269750 -- ^ Name of the scribe
270751 -> LogEnv
271752 -> LogEnv
272−unregisterScribe nm = logEnvScribes . at nm .~ Nothing
753+unregisterScribe nm = logEnvScribes %~ M.delete nm
754+
755+
756+-------------------------------------------------------------------------------
757+-- | Unregister __all__ scribes. Note that this is __not__ for closing or
758+-- finalizing scribes, use 'closeScribes' for that. This mainly only
759+-- makes sense to use with something like MonadReader's @local@
760+-- function to temporarily disavow any loggers for a block of code.
761+clearScribes
762+ :: LogEnv
763+ -> LogEnv
764+clearScribes = logEnvScribes .~ mempty
765+
766+
767+-------------------------------------------------------------------------------
768+-- | Finalize a scribe. The scribe is removed from the environment,
769+-- its finalizer is called so that it can never be written to again
770+-- and all pending writes are flushed. Note that this will throw any
771+-- exceptions yoru finalizer will throw, and that LogEnv is immutable,
772+-- so it will not be removed in that case.
773+closeScribe
774+ :: Text
775+ -- ^ Name of the scribe
776+ -> LogEnv
777+ -> IO LogEnv
778+closeScribe nm le = do
779+ maybe (return ()) (scribeFinalizer . shScribe) (M.lookup nm (_logEnvScribes le))
780+ return (le & logEnvScribes %~ M.delete nm)
781+
273782
783+-------------------------------------------------------------------------------
784+-- | Call this at the end of your program. This is a blocking call
785+-- that stop writing to a scribe's queue, waits for the queue to
786+-- empty, finalizes each scribe in the log environment and then
787+-- removes it. Finalizers are all run even if one of them throws, but
788+-- the exception will be re-thrown at the end.
789+closeScribes
790+ :: LogEnv
791+ -> IO LogEnv
792+closeScribes le = do
793+ -- We want to run every finalizer here so we'll not save
794+ -- intermediate logenvs and just clear scribes at the end.
795+ let actions = [void (closeScribe k le) | k <- M.keys (_logEnvScribes le)]
796+ FT.foldr finally (return ()) actions
797+ return (le & logEnvScribes .~ mempty)
274798
275799
276800-------------------------------------------------------------------------------
277−-- | Monads where katip logging actions can be performed
278−class MonadIO m => Katip m where
801+-- | Monads where katip logging actions can be performed. Katip is the
802+-- most basic logging monad. You will typically use this directly if
803+-- you either don't want to use namespaces/contexts heavily or if you
804+-- want to pass in specific contexts and/or namespaces at each log site.
805+--
806+-- For something more powerful, look at the docs for 'KatipContext',
807+-- which keeps a namespace and merged context. You can write simple
808+-- functions that add additional namespacing and merges additional
809+-- context on the fly.
810+--
811+-- 'localLogEnv' was added to allow for lexically-scoped modifications
812+-- of the log env that are reverted when the supplied monad
813+-- completes. 'katipNoLogging', for example, uses this to temporarily
814+-- pause log outputs.
815+class MonadIO m => Katip m where
279816 getLogEnv :: m LogEnv
817+ localLogEnv :: (LogEnv -> LogEnv) -> m a -> m a
280818
281819
282820instance Katip m => Katip (ReaderT s m) where
283821 getLogEnv = lift getLogEnv
822+ localLogEnv = mapReaderT . localLogEnv
284823
285824
825+#if !MIN_VERSION_either(4, 5, 0)
286826instance Katip m => Katip (EitherT s m) where
287827 getLogEnv = lift getLogEnv
828+ localLogEnv = mapEitherT . localLogEnv
829+#endif
830+
831+
832+instance Katip m => Katip (ExceptT s m) where
833+ getLogEnv = lift getLogEnv
834+ localLogEnv = mapExceptT . localLogEnv
288835
289836
290837instance Katip m => Katip (MaybeT m) where
291838 getLogEnv = lift getLogEnv
839+ localLogEnv = mapMaybeT . localLogEnv
292840
293841
294842instance Katip m => Katip (StateT s m) where
295843 getLogEnv = lift getLogEnv
844+ localLogEnv = mapStateT . localLogEnv
845+
846+
847+instance (Katip m, Monoid w) => Katip (RWST r w s m) where
848+ getLogEnv = lift getLogEnv
849+ localLogEnv = mapRWST . localLogEnv
850+
851+
852+instance (Katip m, Monoid w) => Katip (Strict.RWST r w s m) where
853+ getLogEnv = lift getLogEnv
854+ localLogEnv = Strict.mapRWST . localLogEnv
855+
856+
857+instance Katip m => Katip (Strict.StateT s m) where
858+ getLogEnv = lift getLogEnv
859+ localLogEnv = Strict.mapStateT . localLogEnv
296860
297861
298862instance (Katip m, Monoid s) => Katip (WriterT s m) where
299863 getLogEnv = lift getLogEnv
864+ localLogEnv = mapWriterT . localLogEnv
865+
866+
867+instance (Katip m, Monoid s) => Katip (Strict.WriterT s m) where
868+ getLogEnv = lift getLogEnv
869+ localLogEnv = Strict.mapWriterT . localLogEnv
870+
871+
872+instance (Katip m) => Katip (ResourceT m) where
873+ getLogEnv = lift getLogEnv
874+ localLogEnv = transResourceT . localLogEnv
300875
301876
302877-------------------------------------------------------------------------------
303−-- | A concrete monad you can use to run logging actions.
878+-- | A concrete monad you can use to run logging actions. Use this if
879+-- you prefer an explicit monad transformer stack and adding layers as
880+-- opposed to implementing 'Katip' for your monad.
304881newtype KatipT m a = KatipT { unKatipT :: ReaderT LogEnv m a }
305882 deriving ( Functor, Applicative, Monad, MonadIO
306− , MonadMask, MonadCatch, MonadThrow )
883+ , MonadMask, MonadCatch, MonadThrow, MonadResource, MonadTrans
884+ , MonadBase b)
307885
308886
309887instance MonadIO m => Katip (KatipT m) where
310888 getLogEnv = KatipT ask
311−
889+ localLogEnv f (KatipT m) = KatipT $ local f m
890+
891+
892+instance MonadTransControl KatipT where
893+ type StT (KatipT) a = a
894+ liftWith f = KatipT $ ReaderT $ \le -> f $ \t -> runKatipT le t
895+ restoreT = KatipT . ReaderT . const
896+ {-# INLINABLE liftWith #-}
897+ {-# INLINABLE restoreT #-}
898+
899+
900+instance (MonadBaseControl b m) => MonadBaseControl b (KatipT m) where
901+ type StM ((KatipT) m) a = ComposeSt (KatipT) m a
902+ liftBaseWith = defaultLiftBaseWith
903+ restoreM = defaultRestoreM
904+
905+instance MonadUnliftIO m => MonadUnliftIO (KatipT m) where
906+#if MIN_VERSION_unliftio_core(0, 2, 0)
907+ withRunInIO inner = KatipT $ ReaderT $ \le -> withRunInIO $ \run ->
908+ inner (run . runKatipT le)
909+#else
910+ askUnliftIO = KatipT $
911+ withUnliftIO $ \u ->
912+ pure (UnliftIO (unliftIO u . unKatipT))
913+#endif
914+
915+#if MIN_VERSION_base(4, 9, 0)
916+instance MF.MonadFail m => MF.MonadFail (KatipT m) where
917+ fail msg = lift (MF.fail msg)
918+ {-# INLINE fail #-}
919+#endif
312920
313921-------------------------------------------------------------------------------
314922-- | Execute 'KatipT' on a log env.
@@ -316,38 +924,73 @@ runKatipT :: LogEnv -> KatipT m a -> m a
316924runKatipT le (KatipT f) = runReaderT f le
317925
318926
927+-------------------------------------------------------------------------------
928+-- | Disable all scribes for the given monadic action, then restore
929+-- them afterwards. Works in any Katip monad.
930+katipNoLogging
931+ :: ( Katip m
932+ )
933+ => m a
934+ -> m a
935+katipNoLogging = localLogEnv (\le -> set logEnvScribes mempty le)
936+
937+
319938-------------------------------------------------------------------------------
320939-- | Log with everything, including a source code location. This is
321940-- very low level and you typically can use 'logT' in its place.
322−logI
323− :: (Applicative m, LogContext a, Katip m)
941+logItem
942+ :: (A.Applicative m, LogItem a, Katip m)
324943 => a
325944 -> Namespace
326945 -> Maybe Loc
327946 -> Severity
328947 -> LogStr
329948 -> m ()
330−logI a ns loc sev msg = do
949+logItem a ns loc sev msg = do
950+ LogEnv{..} <- getLogEnv
951+ logKatipItem =<< liftIO
952+ (Item <$> pure _logEnvApp
953+ <*> pure _logEnvEnv
954+ <*> pure sev
955+ <*> (mkThreadIdText <$> myThreadId)
956+ <*> pure _logEnvHost
957+ <*> pure _logEnvPid
958+ <*> pure a
959+ <*> pure msg
960+ <*> _logEnvTimer
961+ <*> pure (_logEnvApp <> ns)
962+ <*> pure loc)
963+
964+-- | Log already constructed 'Item'. This is the lowest level function that other log*
965+-- functions use.
966+-- It can be useful when implementing centralised logging services.
967+logKatipItem
968+ :: (A.Applicative m, LogItem a, Katip m)
969+ => Item a
970+ -> m ()
971+logKatipItem item = do
331972 LogEnv{..} <- getLogEnv
332− item <- Item
333− <$> pure _logEnvNs
334− <*> pure _logEnvEnv
335− <*> pure sev
336− <*> liftIO myThreadId
337− <*> pure _logEnvHost
338− <*> pure _logEnvPid
339− <*> pure a
340− <*> pure msg
341− <*> liftIO _logEnvTimer
342− <*> pure (_logEnvNs <> ns)
343− <*> pure loc
344− liftIO $ forM_ (M.elems _logEnvScribes) $ \ (Scribe h) -> h item
973+ liftIO $
974+ FT.forM_ (M.elems _logEnvScribes) $ \ ScribeHandle {..} -> do
975+ whenM (scribePermitItem shScribe item) $
976+ void $ atomically (tryWriteTBQueue shChan (NewItem item))
977+
978+-------------------------------------------------------------------------------
979+tryWriteTBQueue
980+ :: TBQueue a
981+ -> a
982+ -> STM Bool
983+ -- ^ Did we write?
984+tryWriteTBQueue q a = do
985+ full <- isFullTBQueue q
986+ unless full (writeTBQueue q a)
987+ return (not full)
345988
346989
347990-------------------------------------------------------------------------------
348991-- | Log with full context, but without any code location.
349992logF
350− :: (Applicative m, LogContext a, Katip m)
993+ :: (Applicative m, LogItem a, Katip m)
351994 => a
352995 -- ^ Contextual payload for the log
353996 -> Namespace
@@ -357,18 +1000,37 @@ logF
3571000 -> LogStr
3581001 -- ^ The log message
3591002 -> m ()
360−logF a ns sev msg = logI a ns Nothing sev msg
1003+logF a ns sev msg = logItem a ns Nothing sev msg
1004+
1005+
1006+
1007+-------------------------------------------------------------------------------
1008+-- | Perform an action while logging any exceptions that may occur.
1009+-- Inspired by 'onException`.
1010+--
1011+-- >>>> logException () mempty ErrorS (error "foo")
1012+logException
1013+ :: (Katip m, LogItem a, MonadCatch m, Applicative m)
1014+ => a -- ^ Log context
1015+ -> Namespace -- ^ Namespace
1016+ -> Severity -- ^ Severity
1017+ -> m b -- ^ Main action being run
1018+ -> m b
1019+logException a ns sev action = action `catchAny` \e -> f e >> throwM e
1020+ where
1021+ f e = logF a ns sev (msg e)
1022+ msg e = ls (T.pack "An exception has occurred: ") <> showLS e
3611023
3621024
3631025-------------------------------------------------------------------------------
3641026-- | Log a message without any payload/context or code location.
365−logM
1027+logMsg
3661028 :: (Applicative m, Katip m)
3671029 => Namespace
3681030 -> Severity
3691031 -> LogStr
3701032 -> m ()
371−logM ns sev msg = logF () ns sev msg
1033+logMsg ns sev msg = logF () ns sev msg
3721034
3731035
3741036instance TH.Lift Namespace where
@@ -385,14 +1047,14 @@ instance TH.Lift Verbosity where
3851047
3861048
3871049instance TH.Lift Severity where
388− lift DebugS = [| DebugS |]
389− lift InfoS = [| InfoS |]
390− lift NoticeS = [| NoticeS |]
391− lift WarningS = [| WarningS |]
392− lift ErrorS = [| ErrorS |]
1050+ lift DebugS = [| DebugS |]
1051+ lift InfoS = [| InfoS |]
1052+ lift NoticeS = [| NoticeS |]
1053+ lift WarningS = [| WarningS |]
1054+ lift ErrorS = [| ErrorS |]
3931055 lift CriticalS = [| CriticalS |]
394− lift AlertS = [| AlertS |]
395− lift EmergencyS = [| EmergencyS |]
1056+ lift AlertS = [| AlertS |]
1057+ lift EmergencyS = [| EmergencyS |]
3961058
3971059
3981060-- | Lift a location into an Exp.
@@ -408,17 +1070,69 @@ liftLoc (Loc a b c (d1, d2) (e1, e2)) = [|Loc
4081070
4091071-------------------------------------------------------------------------------
4101072-- | For use when you want to include location in your logs. This will
411−-- fill the 'Maybe Loc' gap in 'logF' of this module.
412−getLoc :: Q Exp
413−getLoc = [| $(location >>= liftLoc) |]
1073+-- fill the 'Maybe Loc' gap in 'logF' of this module, and relies on implicit
1074+-- callstacks when available (GHC > 7.8).
1075+#if MIN_VERSION_base(4, 8, 0)
1076+getLoc :: HasCallStack => Maybe Loc
1077+getLoc = case getCallStack callStack of
1078+ [] -> Nothing
1079+ xs -> Just . toLoc . head $ filter filterKatip xs
1080+ where
1081+ filterKatip :: (String, SrcLoc) -> Bool
1082+ filterKatip (_, srcloc) = not $
1083+ "katip-" `isPrefixOf` srcLocPackage srcloc
1084+
1085+ toLoc :: (String, SrcLoc) -> Loc
1086+ toLoc (_, l) = Loc {
1087+ loc_filename = srcLocFile l
1088+ , loc_package = srcLocPackage l
1089+ , loc_module = srcLocModule l
1090+ , loc_start = (srcLocStartLine l, srcLocStartCol l)
1091+ , loc_end = (srcLocEndLine l, srcLocEndCol l)
1092+ }
1093+#else
1094+getLoc :: Maybe Loc
1095+getLoc = Nothing
1096+#endif
1097+
1098+
1099+-------------------------------------------------------------------------------
1100+-- Like `getLoc`, but uses template-haskell and works with older versions of
1101+-- the compiler (GHC 7.8 or older).
1102+getLocTH :: ExpQ
1103+getLocTH = [| $(location >>= liftLoc) |]
4141104
4151105
4161106-------------------------------------------------------------------------------
417−-- | 'Loc'-tagged logging when using template-haskell is OK.
1107+-- | 'Loc'-tagged logging when using template-haskell.
4181108--
419−-- @$(logT) obj mempty Info "Hello world"@
1109+-- @$(logT) obj mempty InfoS "Hello world"@
4201110logT :: ExpQ
421−logT = [| \ a ns sev msg -> logI a ns (Just $(getLoc)) sev msg |]
1111+logT = [| \ a ns sev msg -> logItem a ns (Just $(getLocTH)) sev msg |]
1112+
1113+
1114+-------------------------------------------------------------------------------
1115+-- | 'Loc'-tagged logging using 'GHC.Stack' when available.
1116+--
1117+-- This function does not require template-haskell as it
1118+-- automatically uses <https://hackage.haskell.org/package/base-4.8.2.0/docs/GHC-Stack.html#v:getCallStack implicit-callstacks>
1119+-- when the code is compiled using GHC > 7.8. Using an older version of the
1120+-- compiler will result in the emission of a log line without any location information,
1121+-- so be aware of it. Users using GHC <= 7.8 may want to use the template-haskell function
1122+-- `logT` for maximum compatibility.
1123+--
1124+-- @logLoc obj mempty InfoS "Hello world"@
1125+#if MIN_VERSION_base(4, 8, 0)
1126+logLoc :: (Applicative m, LogItem a, Katip m, HasCallStack)
1127+#else
1128+logLoc :: (Applicative m, LogItem a, Katip m)
1129+#endif
1130+ => a
1131+ -> Namespace
1132+ -> Severity
1133+ -> LogStr
1134+ -> m ()
1135+logLoc a ns = logItem a ns getLoc
4221136
4231137
4241138-- taken from the file-location package