git.iko.soy/typeable/katip
on master help sign in
More intelligently set up logging when scribe created

This is for #84 1. Do not create an index when sharding is in use and you won't be using it anyway. 2. Update index settings if index already exists. This feature was since added to bloodhound and uses the types to restrict to only the settings possible to update after creation.

commitd7f4097b58519964a97fd9586e24836410987ada authorMichael Xavier <michael@michaelxavier.net> dateFri 9 Mar 2018, 10:26 (9 years ago) parent706a2b4
2 files changed+46−13
compare with master
unifiedsplit
katip-elasticsearch/src/Katip/Scribes/ElasticSearch/Internal.hs+37−13 katip-elasticsearch/test/Main.hs+9
katip-elasticsearch/src/Katip/Scribes/ElasticSearch/Internal.hs+37−13
@@ -28,6 +28,7 @@ import Control.Retry (RetryPolicy,
2828 recovering)
2929import Data.Aeson
3030import Data.ByteString.Lazy (ByteString)
31+import Data.List.NonEmpty (NonEmpty(..))
3132import Data.Monoid ((<>))
3233import Data.Text (Text)
3334import qualified Data.Text as T
@@ -240,7 +241,10 @@ splitTime t = asMins `divMod` 60
240241
241242-------------------------------------------------------------------------------
242243data EsScribeSetupError = CouldNotCreateIndex !(Response ByteString)
243− | CouldNotCreateMapping !(Response ByteString) deriving (Typeable, Show)
244+ | CouldNotUpdateIndexSettings !(Response ByteString)
245+ | CouldNotCreateMapping !(Response ByteString)
246+ | CouldNotPutTemplate !(Response ByteString)
247+ deriving (Typeable, Show)
244248
245249
246250instance Exception EsScribeSetupError
@@ -273,18 +277,26 @@ mkEsScribe cfg@EsScribeCfg {..} env ix mapping sev verb = do
273277 endSig <- newEmptyMVar
274278
275279 runBH prx env $ do
276− chk <- indexExists prx ix
277− -- note that this doesn't update settings. That's not available
278− -- through the Bloodhound API yet
279− unless chk $ void $ do
280− r1 <- createIndex prx essIndexSettings ix
281− unless (statusIsSuccessful (responseStatus r1)) $
282− liftIO $ throwIO (CouldNotCreateIndex r1)
283− r2 <- if shardingEnabled
284− then putTemplate prx tpl tplName
285− else putMapping prx ix mapping base
286− unless (statusIsSuccessful (responseStatus r2)) $
287− liftIO $ throwIO (CouldNotCreateMapping r2)
280+ if shardingEnabled
281+ then do
282+ -- create or update
283+ res <- putTemplate prx tpl tplName
284+ unless (statusIsSuccessful (responseStatus res)) $
285+ liftIO $ throwIO (CouldNotPutTemplate res)
286+ else do
287+ ixExists <- indexExists prx ix
288+ if ixExists
289+ then do
290+ res <- updateIndexSettings prx (toUpdatabaleIndexSettings prx essIndexSettings) ix
291+ unless (statusIsSuccessful (responseStatus res)) $
292+ liftIO $ throwIO (CouldNotUpdateIndexSettings res)
293+ else do
294+ r1 <- createIndex prx essIndexSettings ix
295+ unless (statusIsSuccessful (responseStatus r1)) $
296+ liftIO $ throwIO (CouldNotCreateIndex r1)
297+ r2 <- putMapping prx ix mapping base
298+ unless (statusIsSuccessful (responseStatus r2)) $
299+ liftIO $ throwIO (CouldNotCreateMapping r2)
288300
289301 workers <- replicateM (unEsPoolSize essPoolSize) $ async $
290302 startWorker cfg env mapping q
@@ -436,6 +448,7 @@ class ESVersion v where
436448 type BHEnv v
437449 type IndexSettings v
438450 defaultIndexSettings :: proxy v -> IndexSettings v
451+ type UpdatableIndexSetting v
439452 type IndexName v
440453 toIndexName :: proxy v -> Text -> IndexName v
441454 fromIndexName :: proxy v -> IndexName v -> Text
@@ -454,11 +467,14 @@ class ESVersion v where
454467 type IndexDocumentSettings v
455468 defaultIndexDocumentSettings :: proxy v -> IndexDocumentSettings v
456469
470+ toUpdatabaleIndexSettings :: proxy v -> IndexSettings v -> NonEmpty (UpdatableIndexSetting v)
471+
457472 -- Operations
458473 -- We're deciding on IO here, but it isn't necessary
459474 indexExists :: proxy v -> IndexName v -> BH v IO Bool
460475 indexDocument :: ToJSON doc => proxy v -> IndexName v -> MappingName v -> IndexDocumentSettings v -> doc -> DocId v -> BH v IO (Response ByteString)
461476 createIndex :: proxy v -> IndexSettings v -> IndexName v -> BH v IO (Response ByteString)
477+ updateIndexSettings :: proxy v -> NonEmpty (UpdatableIndexSetting v) -> IndexName v -> BH v IO (Response ByteString)
462478 putTemplate :: proxy v -> IndexTemplate v -> TemplateName v -> BH v IO (Response ByteString)
463479 putMapping :: (ToJSON a) => proxy v -> IndexName v -> MappingName v -> a -> BH v IO (Response ByteString)
464480
@@ -476,6 +492,7 @@ instance ESVersion ESV1 where
476492 type IndexSettings ESV1 = V1.IndexSettings
477493 defaultIndexSettings _ = V1.defaultIndexSettings
478494 type IndexName ESV1 = V1.IndexName
495+ type UpdatableIndexSetting ESV1 = V1.UpdatableIndexSetting
479496 toIndexName _ = V1.IndexName
480497 fromIndexName _ (V1.IndexName x) = x
481498 type MappingName ESV1 = V1.MappingName
@@ -491,10 +508,13 @@ instance ESVersion ESV1 where
491508 type IndexTemplate ESV1 = V1.IndexTemplate
492509 toIndexTemplate _ = V1.IndexTemplate
493510 type IndexDocumentSettings ESV1 = V1.IndexDocumentSettings
511+ toUpdatabaleIndexSettings _ s =
512+ (V1.NumberOfReplicas (V1.indexReplicas s)) :| []
494513 defaultIndexDocumentSettings _ = V1.defaultIndexDocumentSettings
495514 indexExists _ = V1.indexExists
496515 indexDocument _ = V1.indexDocument
497516 createIndex _ = V1.createIndex
517+ updateIndexSettings _ = V1.updateIndexSettings
498518 putTemplate _ = V1.putTemplate
499519 putMapping _ = V1.putMapping
500520 unanalyzedStringType _ = "string"
@@ -506,6 +526,7 @@ data ESV5 = ESV5
506526instance ESVersion ESV5 where
507527 type BHEnv ESV5 = V5.BHEnv
508528 type IndexSettings ESV5 = V5.IndexSettings
529+ type UpdatableIndexSetting ESV5 = V5.UpdatableIndexSetting
509530 defaultIndexSettings _ = V5.defaultIndexSettings
510531 type IndexName ESV5 = V5.IndexName
511532 toIndexName _ = V5.IndexName
@@ -523,10 +544,13 @@ instance ESVersion ESV5 where
523544 type IndexTemplate ESV5 = V5.IndexTemplate
524545 toIndexTemplate _ = V5.IndexTemplate
525546 type IndexDocumentSettings ESV5 = V5.IndexDocumentSettings
547+ toUpdatabaleIndexSettings _ s =
548+ (V5.NumberOfReplicas (V5.indexReplicas s)) :| []
526549 defaultIndexDocumentSettings _ = V5.defaultIndexDocumentSettings
527550 indexExists _ = V5.indexExists
528551 indexDocument _ = V5.indexDocument
529552 createIndex _ = V5.createIndex
553+ updateIndexSettings _ = V5.updateIndexSettings
530554 putTemplate _ = V5.putTemplate
531555 putMapping _ = V5.putMapping
532556 unanalyzedStringType _ = "keyword"
katip-elasticsearch/test/Main.hs+9
@@ -17,6 +17,7 @@ module Main
1717-------------------------------------------------------------------------------
1818import Control.Applicative as A
1919import Control.Concurrent.STM
20+import Control.Exception
2021import Control.Lens hiding (mapping, (.=))
2122import Control.Monad
2223import Control.Monad.IO.Class
@@ -237,6 +238,14 @@ esTests prx = testGroup "elasticsearch scribe"
237238 let l = head logs
238239 l ^? key "_source" . key "msg" . _String @?= Just "A test message"
239240 l ^? key "_source" . key "data" . key "whatever::b" . _Bool @?= Just True
241+ , testCase "can set up twice with no sharding" $ do
242+ let setup = bracket_ (setupSearch prx (\c -> c { essIndexSharding = NoIndexSharding })) (teardownSearch prx)
243+ setup $ setup $
244+ return ()
245+ , testCase "can set up twice with sharding" $ do
246+ let setup = bracket_ (setupSearch prx (\c -> c { essIndexSharding = DailyIndexSharding })) (teardownSearch prx)
247+ setup $ setup $
248+ return ()
240249 , withSearch prx $ \setup -> testCase "date-based index sharding" $ do
241250 let t1 = mkTime 2016 1 2 3 4 5
242251 fakeClock <- newTVarIO t1