From 142e864fb16d86695652d146992e42e1c5df856b Mon Sep 17 00:00:00 2001 From: Andrew Martin Date: Tue, 5 Dec 2017 13:31:17 -0500 Subject: [PATCH 1/2] add ability to change compression codec. This allows us to use DEFLATE instead of lz4. document what compression will actually be used --- src/Database/V5/Bloodhound/Types.hs | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/src/Database/V5/Bloodhound/Types.hs b/src/Database/V5/Bloodhound/Types.hs index 7c50165..8a61fbb 100644 --- a/src/Database/V5/Bloodhound/Types.hs +++ b/src/Database/V5/Bloodhound/Types.hs @@ -74,6 +74,7 @@ module Database.V5.Bloodhound.Types , UpdatableIndexSetting(..) , IndexSettingsSummary(..) , AllocationPolicy(..) + , Compression(..) , ReplicaBounds(..) , Bytes(..) , gigabytes @@ -586,6 +587,7 @@ data UpdatableIndexSetting = NumberOfReplicas ReplicaCount | TTLDisablePurge Bool -- ^ Disables temporarily the purge of expired docs. | TranslogFSType FSType + | CompressionSetting Compression | IndexCompoundFormat CompoundFormat | IndexCompoundOnFlush Bool | WarmerEnabled Bool @@ -688,6 +690,26 @@ data ReplicaBounds = ReplicasBounded Int Int | ReplicasUnbounded deriving (Eq, Read, Show, Generic, Typeable) +data Compression + = CompressionDefault + -- ^ Compress with LZ4 + | CompressionBest + -- ^ Compress with DEFLATE. Elastic + -- + -- that this can reduce disk use by 15%-25%. + deriving (Eq,Show,Generic,Typeable) + +instance ToJSON Compression where + toJSON x = case x of + CompressionDefault -> toJSON ("default" :: Text) + CompressionBest -> toJSON ("best_compression" :: Text) + +instance FromJSON Compression where + parseJSON = withText "Compression" $ \t -> case t of + "default" -> return CompressionDefault + "best_compression" -> return CompressionBest + _ -> fail "invalid compression codec" + -- | A measure of bytes used for various configurations. You may want -- to use smart constructors like 'gigabytes' for larger values. -- @@ -3079,6 +3101,7 @@ instance ToJSON UpdatableIndexSetting where toJSON (GCDeletes x) = oPath ("index" :| ["gc_deletes"]) (NominalDiffTimeJSON x) toJSON (TTLDisablePurge x) = oPath ("index" :| ["ttl", "disable_purge"]) x toJSON (TranslogFSType x) = oPath ("index" :| ["translog", "fs", "type"]) x + toJSON (CompressionSetting x) = oPath ("index" :| ["codec"]) x toJSON (IndexCompoundFormat x) = oPath ("index" :| ["compound_format"]) x toJSON (IndexCompoundOnFlush x) = oPath ("index" :| ["compound_on_flush"]) x toJSON (WarmerEnabled x) = oPath ("index" :| ["warmer", "enabled"]) x @@ -3112,6 +3135,7 @@ instance FromJSON UpdatableIndexSetting where <|> gcDeletes `taggedAt` ["index", "gc_deletes"] <|> ttlDisablePurge `taggedAt` ["index", "ttl", "disable_purge"] <|> translogFSType `taggedAt` ["index", "translog", "fs", "type"] + <|> compressionSetting `taggedAt` ["index", "codec"] <|> compoundFormat `taggedAt` ["index", "compound_format"] <|> compoundOnFlush `taggedAt` ["index", "compound_on_flush"] <|> warmerEnabled `taggedAt` ["index", "warmer", "enabled"] @@ -3146,6 +3170,7 @@ instance FromJSON UpdatableIndexSetting where gcDeletes = pure . GCDeletes . ndtJSON ttlDisablePurge = pure . TTLDisablePurge translogFSType = pure . TranslogFSType + compressionSetting = pure . CompressionSetting compoundFormat = pure . IndexCompoundFormat compoundOnFlush = pure . IndexCompoundOnFlush warmerEnabled = pure . WarmerEnabled From b488c45217c1c8980f262069560796e406c34705 Mon Sep 17 00:00:00 2001 From: Andrew Martin Date: Tue, 5 Dec 2017 14:14:30 -0500 Subject: [PATCH 2/2] add tests for compression --- tests/V5/tests.hs | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/tests/V5/tests.hs b/tests/V5/tests.hs index 64e6e2e..2a66370 100644 --- a/tests/V5/tests.hs +++ b/tests/V5/tests.hs @@ -914,6 +914,7 @@ instance Arbitrary AnalyzerDefinition where arbitrary = sopArbitrary; shrink = g instance Arbitrary Analysis where arbitrary = sopArbitrary; shrink = genericShrink instance Arbitrary Tokenizer where arbitrary = sopArbitrary; shrink = genericShrink instance Arbitrary UpdatableIndexSetting where arbitrary = sopArbitrary; shrink = genericShrink +instance Arbitrary Compression where arbitrary = sopArbitrary; shrink = genericShrink instance Arbitrary Bytes where arbitrary = sopArbitrary; shrink = genericShrink instance Arbitrary AllocationPolicy where arbitrary = sopArbitrary; shrink = genericShrink instance Arbitrary InitialShardCount where arbitrary = sopArbitrary; shrink = genericShrink @@ -1648,6 +1649,25 @@ main = hspec $ do updates ) + it "accepts default compression codec" $ when' (atleast es50) $ withTestEnv $ do + _ <- deleteExampleIndex + let updates = [CompressionSetting CompressionDefault] + createResp <- createIndexWith (updates ++ [NumberOfReplicas (ReplicaCount 0)]) 1 testIndex + liftIO $ validateStatus createResp 200 + getResp <- getIndexSettings testIndex + liftIO $ getResp `shouldBe` Right + (IndexSettingsSummary testIndex (IndexSettings (ShardCount 1) (ReplicaCount 0)) updates) + + it "accepts best compression codec" $ when' (atleast es50) $ withTestEnv $ do + _ <- deleteExampleIndex + let updates = [CompressionSetting CompressionBest] + createResp <- createIndexWith (updates ++ [NumberOfReplicas (ReplicaCount 0)]) 1 testIndex + liftIO $ validateStatus createResp 200 + getResp <- getIndexSettings testIndex + liftIO $ getResp `shouldBe` Right + (IndexSettingsSummary testIndex (IndexSettings (ShardCount 1) (ReplicaCount 0)) updates) + + describe "Index Optimization" $ do it "returns a successful response upon completion" $ withTestEnv $ do _ <- createExampleIndex