1889edd · typeable/bloodhound

git.iko.soy/typeable/bloodhound

on master help sign in

Implement index optimization

1889eddMichael Xavier11 years ago

src/Database/Bloodhound/Client.hs+47−2

compare with master

unifiedsplit

@@ -1,5 +1,6 @@

1{-# LANGUAGE OverloadedStrings #-}

2{-# LANGUAGE RecordWildCards #-}

3+{-# LANGUAGE TupleSections #-}

4-------------------------------------------------------------------------------

5-------------------------------------------------------------------------------

5-- |

6-- |

@@ -26,6 +27,7 @@ module Database.Bloodhound.Client

26 , deleteIndex

27 , deleteIndex

27 , updateIndexSettings

28 , updateIndexSettings

28 , getIndexSettings

29 , getIndexSettings

30+ , optimizeIndex

29 , indexExists

31 , indexExists

30 , openIndex

32 , openIndex

31 , closeIndex

33 , closeIndex

@@ -81,7 +83,7 @@ import qualified Data.HashMap.Strict as HM

81import Data.Ix

83import Data.Ix

82import qualified Data.List as LS (filter, foldl')

84import qualified Data.List as LS (filter, foldl')

83import Data.List.NonEmpty (NonEmpty (..))

85import Data.List.NonEmpty (NonEmpty (..))

84−import Data.Maybe (fromMaybe, isJust)

86+import Data.Maybe (catMaybes, fromMaybe, isJust)

85import Data.Monoid

87import Data.Monoid

86import Data.Text (Text)

88import Data.Text (Text)

87import qualified Data.Text as T

89import qualified Data.Text as T

@@ -302,6 +304,48 @@ getIndexSettings (IndexName indexName) = do

302 where url = joinPath [indexName, "_settings"]

304 where url = joinPath [indexName, "_settings"]

303

305

304

306

307+-- | 'optimizeIndex' will optimize a single index, list of indexes or

308+-- all indexes. Note that this call will block until finishing but

309+-- will continue even if the request times out. Concurrent requests to

310+-- optimize an index while another is performing will block until the

311+-- previous one finishes. For more information see

312+-- <https://www.elastic.co/guide/en/elasticsearch/reference/1.7/indices-optimize.html>. Nothing

313+-- worthwhile comes back in the reply body, so matching on the status

314+-- should suffice.

315+--

316+-- 'optimizeIndex' with a maxNumSegments of 1 and onlyExpungeDeletes

317+-- to True is the main way to release disk space back to the OS being

318+-- held by deleted documents.

319+--

320+-- Note that this API was deprecated in ElasticSearch 2.1 for the

321+-- almost completely identical forcemerge API. Adding support to that

322+-- API would be trivial but due to the significant breaking changes,

323+-- this library cannot currently be used with >= 2.0, so that feature was omitted.

324+--

325+-- >>> let ixn = IndexName "unoptimizedindex"

326+-- >>> _ <- runBH' $ deleteIndex ixn >> createIndex defaultIndexSettings ixn

327+-- >>> response <- runBH' $ optimizeIndex (IndexList (ixn :| [])) (defaultIndexOptimizationSettings { maxNumSegments = Just 1, onlyExpungeDeletes = True })

328+-- >>> respIsTwoHunna response

329+-- True

330+optimizeIndex :: MonadBH m => IndexSelection -> IndexOptimizationSettings -> m Reply

331+optimizeIndex ixs IndexOptimizationSettings {..} =

332+ bindM2 post url (return body)

333+ where url = addQuery params <$> joinPath [indexName, "_optimize"]

334+ params = catMaybes [ ("max_num_segments",) . Just . showText <$> maxNumSegments

335+ , Just ("only_expunge_deletes", Just (boolQP onlyExpungeDeletes))

336+ , Just ("flush", Just (boolQP flushAfterOptimize))

337+ ]

338+ indexName = indexSelectionName ixs

339+ boolQP True = "true"

340+ boolQP False = "false"

341+ body = Nothing

342+

343+

344+-------------------------------------------------------------------------------

345+indexSelectionName :: IndexSelection -> Text

346+indexSelectionName (IndexList names) = T.intercalate "," [n | IndexName n <- toList names]

347+indexSelectionName AllIndexes = "_all"

348+

305deepMerge :: [Object] -> Object

349deepMerge :: [Object] -> Object

306deepMerge = LS.foldl' go mempty

350deepMerge = LS.foldl' go mempty

307 where go acc = LS.foldl' go' acc . HM.toList

351 where go acc = LS.foldl' go' acc . HM.toList

@@ -395,6 +439,7 @@ closeIndex = openOrCloseIndexes CloseIndex

395-- >>> let aliasName = IndexName "an-alias"

439-- >>> let aliasName = IndexName "an-alias"

396-- >>> let iAlias = IndexAlias src (IndexAliasName aliasName)

440-- >>> let iAlias = IndexAlias src (IndexAliasName aliasName)

397-- >>> let aliasCreate = IndexAliasCreate Nothing Nothing

441-- >>> let aliasCreate = IndexAliasCreate Nothing Nothing

442+-- >>> _ <- runBH' $ deleteIndex src

398-- >>> respIsTwoHunna <$> runBH' (createIndex defaultIndexSettings src)

443-- >>> respIsTwoHunna <$> runBH' (createIndex defaultIndexSettings src)

399-- True

444-- True

400-- >>> runBH' $ indexExists src

445-- >>> runBH' $ indexExists src

@@ -484,7 +529,7 @@ versionCtlParams cfg =

484 ExternalGTE (ExternalDocVersion v) -> versionParams v "external_gte"

529 ExternalGTE (ExternalDocVersion v) -> versionParams v "external_gte"

485 ForceVersion (ExternalDocVersion v) -> versionParams v "force"

530 ForceVersion (ExternalDocVersion v) -> versionParams v "force"

486 where

531 where

487− vt = T.pack . show . docVersionNumber

532+ vt = showText . docVersionNumber

488 versionParams v t = [ ("version", Just $ vt v)

533 versionParams v t = [ ("version", Just $ vt v)

489 , ("version_type", Just t)

534 , ("version_type", Just t)

490 ]

535 ]

src/Database/Bloodhound/Types.hs+31

@@ -140,6 +140,9 @@ module Database.Bloodhound.Types

140 , FieldName(..)

141 , Script(..)

142 , IndexName(..)

143+ , IndexSelection(..)

144+ , IndexOptimizationSettings(..)

145+ , defaultIndexOptimizationSettings

143 , TemplateName(..)

146 , TemplateName(..)

144 , TemplatePattern(..)

147 , TemplatePattern(..)

145 , MappingName(..)

148 , MappingName(..)

@@ -381,6 +384,28 @@ data IndexSettings =

381defaultIndexSettings :: IndexSettings

384defaultIndexSettings :: IndexSettings

382defaultIndexSettings = IndexSettings (ShardCount 3) (ReplicaCount 2)

385defaultIndexSettings = IndexSettings (ShardCount 3) (ReplicaCount 2)

383

386

387+

388+{-| 'IndexOptimizationSettings' is used to configure index optimization. See

389+ <https://www.elastic.co/guide/en/elasticsearch/reference/1.7/indices-optimize.html>

390+ for more info.

391+-}

392+data IndexOptimizationSettings =

393+ IndexOptimizationSettings { maxNumSegments :: Maybe Int

394+ -- ^ Number of segments to optimize to. 1 will fully optimize the index. If omitted, the default behavior is to only optimize if the server deems it necessary.

395+ , onlyExpungeDeletes :: Bool

396+ -- ^ Should the optimize process only expunge segments with deletes in them? If the purpose of the optimization is to free disk space, this should be set to True.

397+ , flushAfterOptimize :: Bool

398+ -- ^ Should a flush be performed after the optimize.

399+ } deriving (Eq, Show, Generic, Typeable)

400+

401+

402+{-| 'defaultIndexOptimizationSettings' implements the default settings that

403+ ElasticSearch uses for index optimization. 'maxNumSegments' is Nothing,

404+ 'onlyExpungeDeletes' is False, and flushAfterOptimize is True.

405+-}

406+defaultIndexOptimizationSettings :: IndexOptimizationSettings

407+defaultIndexOptimizationSettings = IndexOptimizationSettings Nothing False True

408+

384{-| 'UpdatableIndexSetting' are settings which may be updated after an index is created.

409{-| 'UpdatableIndexSetting' are settings which may be updated after an index is created.

385

410

386 <https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-update-settings.html>

411 <https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-update-settings.html>

@@ -764,6 +789,12 @@ newtype ReplicaCount = ReplicaCount Int deriving (Eq, Show, Generic, ToJSON, Typ

764-}

789-}

765newtype IndexName = IndexName Text deriving (Eq, Generic, Show, ToJSON, FromJSON, Typeable)

790newtype IndexName = IndexName Text deriving (Eq, Generic, Show, ToJSON, FromJSON, Typeable)

766

791

792+{-| 'IndexSelection' is used for APIs which take a single index, a list of

793+ indexes, or the special @_all@ index.

794+-}

795+data IndexSelection = IndexList (NonEmpty IndexName)

796+ | AllIndexes deriving (Eq, Generic, Show, Typeable)

797+

767{-| 'TemplateName' is used to describe which template to query/create/delete

798{-| 'TemplateName' is used to describe which template to query/create/delete

768-}

799-}

769newtype TemplateName = TemplateName Text deriving (Eq, Show, Generic, ToJSON, FromJSON, Typeable)

800newtype TemplateName = TemplateName Text deriving (Eq, Show, Generic, ToJSON, FromJSON, Typeable)

tests/tests.hs+6

@@ -1416,6 +1416,12 @@ main = hspec $ do

1416 (IndexSettings (ShardCount 1) (ReplicaCount 0))

1417 (NE.toList updates))

1418

1419+ describe "Index Optimization" $ do

1420+ it "returns a successful response upon completion" $ withTestEnv $ do

1421+ _ <- createExampleIndex

1422+ resp <- optimizeIndex (IndexList (testIndex :| [])) defaultIndexOptimizationSettings

1423+ liftIO $ validateStatus resp 200

1424+

1419 describe "JSON instances" $ do

1425 describe "JSON instances" $ do

1420 propJSON (Proxy :: Proxy Version)

1426 propJSON (Proxy :: Proxy Version)

1421 propJSON (Proxy :: Proxy IndexName)

1427 propJSON (Proxy :: Proxy IndexName)