git.iko.soy/typeable/bloodhound
on master help sign in
Merge pull request #216 from andrewthad/best_compression

add ability to change compression codec. This allows us to use DEFLATE instead of lz4.

commitce723c2089403240dda31eb43bc8b32ea7f79fc1 authorChris Allen <cma@bitemyapp.com> dateTue 5 Dec 2017, 14:14 (9 years ago) parent1e14794b488c45
2 files changed+45
compare with master
unifiedsplit
src/Database/V5/Bloodhound/Types.hs+25 tests/V5/tests.hs+20
src/Database/V5/Bloodhound/Types.hs+25
@@ -74,6 +74,7 @@ module Database.V5.Bloodhound.Types
7474 , UpdatableIndexSetting(..)
7575 , IndexSettingsSummary(..)
7676 , AllocationPolicy(..)
77+ , Compression(..)
7778 , ReplicaBounds(..)
7879 , Bytes(..)
7980 , gigabytes
@@ -586,6 +587,7 @@ data UpdatableIndexSetting = NumberOfReplicas ReplicaCount
586587 | TTLDisablePurge Bool
587588 -- ^ Disables temporarily the purge of expired docs.
588589 | TranslogFSType FSType
590+ | CompressionSetting Compression
589591 | IndexCompoundFormat CompoundFormat
590592 | IndexCompoundOnFlush Bool
591593 | WarmerEnabled Bool
@@ -688,6 +690,26 @@ data ReplicaBounds = ReplicasBounded Int Int
688690 | ReplicasUnbounded
689691 deriving (Eq, Read, Show, Generic, Typeable)
690692
693+data Compression
694+ = CompressionDefault
695+ -- ^ Compress with LZ4
696+ | CompressionBest
697+ -- ^ Compress with DEFLATE. Elastic
698+ -- <https://www.elastic.co/blog/elasticsearch-storage-the-true-story-2.0 blogs>
699+ -- that this can reduce disk use by 15%-25%.
700+ deriving (Eq,Show,Generic,Typeable)
701+
702+instance ToJSON Compression where
703+ toJSON x = case x of
704+ CompressionDefault -> toJSON ("default" :: Text)
705+ CompressionBest -> toJSON ("best_compression" :: Text)
706+
707+instance FromJSON Compression where
708+ parseJSON = withText "Compression" $ \t -> case t of
709+ "default" -> return CompressionDefault
710+ "best_compression" -> return CompressionBest
711+ _ -> fail "invalid compression codec"
712+
691713-- | A measure of bytes used for various configurations. You may want
692714-- to use smart constructors like 'gigabytes' for larger values.
693715--
@@ -3079,6 +3101,7 @@ instance ToJSON UpdatableIndexSetting where
30793101 toJSON (GCDeletes x) = oPath ("index" :| ["gc_deletes"]) (NominalDiffTimeJSON x)
30803102 toJSON (TTLDisablePurge x) = oPath ("index" :| ["ttl", "disable_purge"]) x
30813103 toJSON (TranslogFSType x) = oPath ("index" :| ["translog", "fs", "type"]) x
3104+ toJSON (CompressionSetting x) = oPath ("index" :| ["codec"]) x
30823105 toJSON (IndexCompoundFormat x) = oPath ("index" :| ["compound_format"]) x
30833106 toJSON (IndexCompoundOnFlush x) = oPath ("index" :| ["compound_on_flush"]) x
30843107 toJSON (WarmerEnabled x) = oPath ("index" :| ["warmer", "enabled"]) x
@@ -3112,6 +3135,7 @@ instance FromJSON UpdatableIndexSetting where
31123135 <|> gcDeletes `taggedAt` ["index", "gc_deletes"]
31133136 <|> ttlDisablePurge `taggedAt` ["index", "ttl", "disable_purge"]
31143137 <|> translogFSType `taggedAt` ["index", "translog", "fs", "type"]
3138+ <|> compressionSetting `taggedAt` ["index", "codec"]
31153139 <|> compoundFormat `taggedAt` ["index", "compound_format"]
31163140 <|> compoundOnFlush `taggedAt` ["index", "compound_on_flush"]
31173141 <|> warmerEnabled `taggedAt` ["index", "warmer", "enabled"]
@@ -3146,6 +3170,7 @@ instance FromJSON UpdatableIndexSetting where
31463170 gcDeletes = pure . GCDeletes . ndtJSON
31473171 ttlDisablePurge = pure . TTLDisablePurge
31483172 translogFSType = pure . TranslogFSType
3173+ compressionSetting = pure . CompressionSetting
31493174 compoundFormat = pure . IndexCompoundFormat
31503175 compoundOnFlush = pure . IndexCompoundOnFlush
31513176 warmerEnabled = pure . WarmerEnabled
tests/V5/tests.hs+20
@@ -914,6 +914,7 @@ instance Arbitrary AnalyzerDefinition where arbitrary = sopArbitrary; shrink = g
914914instance Arbitrary Analysis where arbitrary = sopArbitrary; shrink = genericShrink
915915instance Arbitrary Tokenizer where arbitrary = sopArbitrary; shrink = genericShrink
916916instance Arbitrary UpdatableIndexSetting where arbitrary = sopArbitrary; shrink = genericShrink
917+instance Arbitrary Compression where arbitrary = sopArbitrary; shrink = genericShrink
917918instance Arbitrary Bytes where arbitrary = sopArbitrary; shrink = genericShrink
918919instance Arbitrary AllocationPolicy where arbitrary = sopArbitrary; shrink = genericShrink
919920instance Arbitrary InitialShardCount where arbitrary = sopArbitrary; shrink = genericShrink
@@ -1648,6 +1649,25 @@ main = hspec $ do
16481649 updates
16491650 )
16501651
1652+ it "accepts default compression codec" $ when' (atleast es50) $ withTestEnv $ do
1653+ _ <- deleteExampleIndex
1654+ let updates = [CompressionSetting CompressionDefault]
1655+ createResp <- createIndexWith (updates ++ [NumberOfReplicas (ReplicaCount 0)]) 1 testIndex
1656+ liftIO $ validateStatus createResp 200
1657+ getResp <- getIndexSettings testIndex
1658+ liftIO $ getResp `shouldBe` Right
1659+ (IndexSettingsSummary testIndex (IndexSettings (ShardCount 1) (ReplicaCount 0)) updates)
1660+
1661+ it "accepts best compression codec" $ when' (atleast es50) $ withTestEnv $ do
1662+ _ <- deleteExampleIndex
1663+ let updates = [CompressionSetting CompressionBest]
1664+ createResp <- createIndexWith (updates ++ [NumberOfReplicas (ReplicaCount 0)]) 1 testIndex
1665+ liftIO $ validateStatus createResp 200
1666+ getResp <- getIndexSettings testIndex
1667+ liftIO $ getResp `shouldBe` Right
1668+ (IndexSettingsSummary testIndex (IndexSettings (ShardCount 1) (ReplicaCount 0)) updates)
1669+
1670+
16511671 describe "Index Optimization" $ do
16521672 it "returns a successful response upon completion" $ withTestEnv $ do
16531673 _ <- createExampleIndex