git.iko.soy/typeable/bloodhound
on master help sign in
Merge branch 'master' into directGenerators
commit624b07bfb8591703c6b371d652c8f1944e0a4106 authorJosh Berman <berman.josh@gmail.com> dateMon 11 Dec 2017, 14:05 (9 years ago) parent7e292e2ce723c2
3 files changed+206−7
src/Database/V5/Bloodhound/Client.hs+27−2 src/Database/V5/Bloodhound/Types.hs+133−5 tests/V5/tests.hs+46
src/Database/V5/Bloodhound/Client.hs+27−2
@@ -25,6 +25,7 @@ module Database.V5.Bloodhound.Client
2525 withBH
2626 -- ** Indices
2727 , createIndex
28+ , createIndexWith
2829 , deleteIndex
2930 , updateIndexSettings
3031 , getIndexSettings
@@ -512,6 +513,28 @@ createIndex indexSettings (IndexName indexName) =
512513 where url = joinPath [indexName]
513514 body = Just $ encode indexSettings
514515
516+-- | Create an index, providing it with any number of settings. This
517+-- is more expressive than 'createIndex' but makes is more verbose
518+-- for the common case of configuring only the shard count and
519+-- replica count.
520+createIndexWith :: MonadBH m
521+ => [UpdatableIndexSetting]
522+ -> Int -- ^ shard count
523+ -> IndexName
524+ -> m Reply
525+createIndexWith updates shards (IndexName indexName) =
526+ bindM2 put url (return (Just body))
527+ where url = joinPath [indexName]
528+ body = encode $ object
529+ ["settings" .= deepMerge
530+ ( HM.singleton "index.number_of_shards" (toJSON shards) :
531+ [u | Object u <- toJSON <$> updates]
532+ )
533+ ]
534+
535+oPath :: ToJSON a => NonEmpty Text -> a -> Value
536+oPath (k :| []) v = object [k .= v]
537+oPath (k:| (h:t)) v = object [k .= oPath (h :| t) v]
515538
516539-- | 'deleteIndex' will delete an index given a 'Server', and an 'IndexName'.
517540--
@@ -545,7 +568,6 @@ getIndexSettings (IndexName indexName) = do
545568 parseEsResponse =<< get =<< url
546569 where url = joinPath [indexName, "_settings"]
547570
548−
549571-- | 'forceMergeIndex'
550572--
551573-- The force merge API allows to force merging of one or more indices through
@@ -787,7 +809,10 @@ versionCtlParams cfg =
787809-- | 'indexDocument' is the primary way to save a single document in
788810-- Elasticsearch. The document itself is simply something we can
789811-- convert into a JSON 'Value'. The 'DocId' will function as the
790−-- primary key for the document.
812+-- primary key for the document. You are encouraged to generate
813+-- your own id's and not rely on ElasticSearch's automatic id
814+-- generation. Read more about it here:
815+-- https://github.com/bitemyapp/bloodhound/issues/107
791816--
792817-- >>> resp <- runBH' $ indexDocument testIndex testMapping defaultIndexDocumentSettings exampleTweet (DocId "1")
793818-- >>> print resp
src/Database/V5/Bloodhound/Types.hs+133−5
@@ -74,6 +74,7 @@ module Database.V5.Bloodhound.Types
7474 , UpdatableIndexSetting(..)
7575 , IndexSettingsSummary(..)
7676 , AllocationPolicy(..)
77+ , Compression(..)
7778 , ReplicaBounds(..)
7879 , Bytes(..)
7980 , gigabytes
@@ -202,6 +203,7 @@ module Database.V5.Bloodhound.Types
202203 , ZeroTermsQuery(..)
203204 , CutoffFrequency(..)
204205 , Analyzer(..)
206+ , Tokenizer(..)
205207 , MaxExpansions(..)
206208 , Lenient(..)
207209 , MatchQueryType(..)
@@ -376,6 +378,12 @@ module Database.V5.Bloodhound.Types
376378
377379 , EsUsername(..)
378380 , EsPassword(..)
381+
382+ , Analysis(..)
383+ , AnalyzerDefinition(..)
384+ , TokenizerDefinition(..)
385+ , Ngram(..)
386+ , TokenChar(..)
379387 ) where
380388
381389import Control.Applicative as A
@@ -579,12 +587,94 @@ data UpdatableIndexSetting = NumberOfReplicas ReplicaCount
579587 | TTLDisablePurge Bool
580588 -- ^ Disables temporarily the purge of expired docs.
581589 | TranslogFSType FSType
590+ | CompressionSetting Compression
582591 | IndexCompoundFormat CompoundFormat
583592 | IndexCompoundOnFlush Bool
584593 | WarmerEnabled Bool
585594 | MappingTotalFieldsLimit Int
595+ | AnalysisSetting Analysis
596+ -- ^ Analysis is not a dynamic setting and can only be performed on a closed index.
586597 deriving (Eq, Show, Generic, Typeable)
587598
599+data Analysis = Analysis
600+ { analysisAnalyzer :: M.Map Text AnalyzerDefinition
601+ , analysisTokenizer :: M.Map Text TokenizerDefinition
602+ } deriving (Eq,Show,Generic,Typeable)
603+
604+instance ToJSON Analysis where
605+ toJSON (Analysis analyzer tokenizer) = object
606+ [ "analyzer" .= analyzer
607+ , "tokenizer" .= tokenizer
608+ ]
609+
610+instance FromJSON Analysis where
611+ parseJSON = withObject "Analysis" $ \m -> Analysis
612+ <$> m .: "analyzer"
613+ <*> m .: "tokenizer"
614+
615+data AnalyzerDefinition = AnalyzerDefinition
616+ { analyzerDefinitionTokenizer :: Maybe Tokenizer
617+ } deriving (Eq,Show,Generic,Typeable)
618+
619+instance ToJSON AnalyzerDefinition where
620+ toJSON (AnalyzerDefinition tokenizer) = object $ catMaybes
621+ [ fmap ("tokenizer" .=) tokenizer
622+ ]
623+
624+instance FromJSON AnalyzerDefinition where
625+ parseJSON = withObject "AnalyzerDefinition" $ \m -> AnalyzerDefinition
626+ <$> m .:? "tokenizer"
627+
628+
629+data TokenizerDefinition
630+ = TokenizerDefinitionNgram Ngram
631+ deriving (Eq,Show,Generic,Typeable)
632+
633+instance ToJSON TokenizerDefinition where
634+ toJSON x = case x of
635+ TokenizerDefinitionNgram (Ngram minGram maxGram tokenChars) -> object
636+ [ "type" .= ("ngram" :: Text)
637+ , "min_gram" .= minGram
638+ , "max_gram" .= maxGram
639+ , "token_chars" .= tokenChars
640+ ]
641+
642+instance FromJSON TokenizerDefinition where
643+ parseJSON = withObject "TokenizerDefinition" $ \m -> do
644+ typ <- m .: "type" :: Parser Text
645+ case typ of
646+ "ngram" -> fmap TokenizerDefinitionNgram $ Ngram
647+ <$> (fmap unStringlyTypedInt (m .: "min_gram"))
648+ <*> (fmap unStringlyTypedInt (m .: "max_gram"))
649+ <*> m .: "token_chars"
650+ _ -> fail "invalid TokenizerDefinition"
651+
652+data Ngram = Ngram
653+ { ngramMinGram :: Int
654+ , ngramMaxGram :: Int
655+ , ngramTokenChars :: [TokenChar]
656+ } deriving (Eq,Show,Generic,Typeable)
657+
658+data TokenChar = TokenLetter | TokenDigit | TokenWhitespace | TokenPunctuation | TokenSymbol
659+ deriving (Eq,Read,Show,Generic,Typeable)
660+
661+instance ToJSON TokenChar where
662+ toJSON t = String $ case t of
663+ TokenLetter -> "letter"
664+ TokenDigit -> "digit"
665+ TokenWhitespace -> "whitespace"
666+ TokenPunctuation -> "punctuation"
667+ TokenSymbol -> "symbol"
668+
669+instance FromJSON TokenChar where
670+ parseJSON = withText "TokenChar" $ \t -> case t of
671+ "letter" -> return TokenLetter
672+ "digit" -> return TokenDigit
673+ "whitespace" -> return TokenWhitespace
674+ "punctuation" -> return TokenPunctuation
675+ "symbol" -> return TokenSymbol
676+ _ -> fail "invalid TokenChar"
677+
588678data AllocationPolicy = AllocAll
589679 -- ^ Allows shard allocation for all shards.
590680 | AllocPrimaries
@@ -600,6 +690,26 @@ data ReplicaBounds = ReplicasBounded Int Int
600690 | ReplicasUnbounded
601691 deriving (Eq, Read, Show, Generic, Typeable)
602692
693+data Compression
694+ = CompressionDefault
695+ -- ^ Compress with LZ4
696+ | CompressionBest
697+ -- ^ Compress with DEFLATE. Elastic
698+ -- <https://www.elastic.co/blog/elasticsearch-storage-the-true-story-2.0 blogs>
699+ -- that this can reduce disk use by 15%-25%.
700+ deriving (Eq,Show,Generic,Typeable)
701+
702+instance ToJSON Compression where
703+ toJSON x = case x of
704+ CompressionDefault -> toJSON ("default" :: Text)
705+ CompressionBest -> toJSON ("best_compression" :: Text)
706+
707+instance FromJSON Compression where
708+ parseJSON = withText "Compression" $ \t -> case t of
709+ "default" -> return CompressionDefault
710+ "best_compression" -> return CompressionBest
711+ _ -> fail "invalid compression codec"
712+
603713-- | A measure of bytes used for various configurations. You may want
604714-- to use smart constructors like 'gigabytes' for larger values.
605715--
@@ -1020,6 +1130,8 @@ newtype CutoffFrequency =
10201130 CutoffFrequency Double deriving (Eq, Read, Show, Generic, ToJSON, FromJSON, Typeable)
10211131newtype Analyzer =
10221132 Analyzer Text deriving (Eq, Read, Show, Generic, ToJSON, FromJSON, Typeable)
1133+newtype Tokenizer =
1134+ Tokenizer Text deriving (Eq, Read, Show, Generic, ToJSON, FromJSON, Typeable)
10231135newtype MaxExpansions =
10241136 MaxExpansions Int deriving (Eq, Read, Show, Generic, ToJSON, FromJSON, Typeable)
10251137
@@ -1438,13 +1550,15 @@ data MatchQuery =
14381550 , matchQueryAnalyzer :: Maybe Analyzer
14391551 , matchQueryMaxExpansions :: Maybe MaxExpansions
14401552 , matchQueryLenient :: Maybe Lenient
1441− , matchQueryBoost :: Maybe Boost } deriving (Eq, Read, Show, Generic, Typeable)
1553+ , matchQueryBoost :: Maybe Boost
1554+ , matchQueryMinimumShouldMatch :: Maybe Text
1555+ } deriving (Eq, Read, Show, Generic, Typeable)
14421556
14431557{-| 'mkMatchQuery' is a convenience function that defaults the less common parameters,
14441558 enabling you to provide only the 'FieldName' and 'QueryString' to make a 'MatchQuery'
14451559-}
14461560mkMatchQuery :: FieldName -> QueryString -> MatchQuery
1447−mkMatchQuery field query = MatchQuery field query Or ZeroTermsNone Nothing Nothing Nothing Nothing Nothing Nothing
1561+mkMatchQuery field query = MatchQuery field query Or ZeroTermsNone Nothing Nothing Nothing Nothing Nothing Nothing Nothing
14481562
14491563data MatchQueryType =
14501564 MatchPhrase
@@ -2834,7 +2948,9 @@ instance ToJSON MatchQuery where
28342948 toJSON (MatchQuery (FieldName fieldName)
28352949 (QueryString mqQueryString) booleanOperator
28362950 zeroTermsQuery cutoffFrequency matchQueryType
2837− analyzer maxExpansions lenient boost) =
2951+ analyzer maxExpansions lenient boost
2952+ minShouldMatch
2953+ ) =
28382954 object [ fieldName .= omitNulls base ]
28392955 where base = [ "query" .= mqQueryString
28402956 , "operator" .= booleanOperator
@@ -2844,7 +2960,9 @@ instance ToJSON MatchQuery where
28442960 , "analyzer" .= analyzer
28452961 , "max_expansions" .= maxExpansions
28462962 , "lenient" .= lenient
2847− , "boost" .= boost ]
2963+ , "boost" .= boost
2964+ , "minimum_should_match" .= minShouldMatch
2965+ ]
28482966
28492967instance FromJSON MatchQuery where
28502968 parseJSON = withObject "MatchQuery" parse
@@ -2859,6 +2977,7 @@ instance FromJSON MatchQuery where
28592977 <*> o .:? "max_expansions"
28602978 <*> o .:? "lenient"
28612979 <*> o .:? "boost"
2980+ <*> o .:? "minimum_should_match"
28622981
28632982instance ToJSON MultiMatchQuery where
28642983 toJSON (MultiMatchQuery fields (QueryString query) boolOp
@@ -2982,6 +3101,7 @@ instance ToJSON UpdatableIndexSetting where
29823101 toJSON (GCDeletes x) = oPath ("index" :| ["gc_deletes"]) (NominalDiffTimeJSON x)
29833102 toJSON (TTLDisablePurge x) = oPath ("index" :| ["ttl", "disable_purge"]) x
29843103 toJSON (TranslogFSType x) = oPath ("index" :| ["translog", "fs", "type"]) x
3104+ toJSON (CompressionSetting x) = oPath ("index" :| ["codec"]) x
29853105 toJSON (IndexCompoundFormat x) = oPath ("index" :| ["compound_format"]) x
29863106 toJSON (IndexCompoundOnFlush x) = oPath ("index" :| ["compound_on_flush"]) x
29873107 toJSON (WarmerEnabled x) = oPath ("index" :| ["warmer", "enabled"]) x
@@ -2990,6 +3110,7 @@ instance ToJSON UpdatableIndexSetting where
29903110 toJSON (BlocksWrite x) = oPath ("blocks" :| ["write"]) x
29913111 toJSON (BlocksMetaData x) = oPath ("blocks" :| ["metadata"]) x
29923112 toJSON (MappingTotalFieldsLimit x) = oPath ("index" :| ["mapping","total_fields","limit"]) x
3113+ toJSON (AnalysisSetting x) = oPath ("index" :| ["analysis"]) x
29933114
29943115instance FromJSON UpdatableIndexSetting where
29953116 parseJSON = withObject "UpdatableIndexSetting" parse
@@ -3014,6 +3135,7 @@ instance FromJSON UpdatableIndexSetting where
30143135 <|> gcDeletes `taggedAt` ["index", "gc_deletes"]
30153136 <|> ttlDisablePurge `taggedAt` ["index", "ttl", "disable_purge"]
30163137 <|> translogFSType `taggedAt` ["index", "translog", "fs", "type"]
3138+ <|> compressionSetting `taggedAt` ["index", "codec"]
30173139 <|> compoundFormat `taggedAt` ["index", "compound_format"]
30183140 <|> compoundOnFlush `taggedAt` ["index", "compound_on_flush"]
30193141 <|> warmerEnabled `taggedAt` ["index", "warmer", "enabled"]
@@ -3022,6 +3144,7 @@ instance FromJSON UpdatableIndexSetting where
30223144 <|> blocksWrite `taggedAt` ["blocks", "write"]
30233145 <|> blocksMetaData `taggedAt` ["blocks", "metadata"]
30243146 <|> mappingTotalFieldsLimit `taggedAt` ["index", "mapping", "total_fields", "limit"]
3147+ <|> analysisSetting `taggedAt` ["index", "analysis"]
30253148 where taggedAt f ks = taggedAt' f (Object o) ks
30263149 taggedAt' f v [] = f =<< (parseJSON v <|> (parseJSON (unStringlyTypeJSON v)))
30273150 taggedAt' f v (k:ks) = withObject "Object" (\o -> do v' <- o .: k
@@ -3047,6 +3170,7 @@ instance FromJSON UpdatableIndexSetting where
30473170 gcDeletes = pure . GCDeletes . ndtJSON
30483171 ttlDisablePurge = pure . TTLDisablePurge
30493172 translogFSType = pure . TranslogFSType
3173+ compressionSetting = pure . CompressionSetting
30503174 compoundFormat = pure . IndexCompoundFormat
30513175 compoundOnFlush = pure . IndexCompoundOnFlush
30523176 warmerEnabled = pure . WarmerEnabled
@@ -3055,6 +3179,7 @@ instance FromJSON UpdatableIndexSetting where
30553179 blocksWrite = pure . BlocksWrite
30563180 blocksMetaData = pure . BlocksMetaData
30573181 mappingTotalFieldsLimit = pure . MappingTotalFieldsLimit
3182+ analysisSetting = pure . AnalysisSetting
30583183
30593184instance FromJSON IndexSettingsSummary where
30603185 parseJSON = withObject "IndexSettingsSummary" parse
@@ -4643,10 +4768,13 @@ instance FromJSON NodeDataPathStats where
46434768
46444769newtype StringlyTypedDouble = StringlyTypedDouble { unStringlyTypedDouble :: Double }
46454770
4646−
46474771instance FromJSON StringlyTypedDouble where
46484772 parseJSON = fmap StringlyTypedDouble . parseJSON . unStringlyTypeJSON
46494773
4774+newtype StringlyTypedInt = StringlyTypedInt { unStringlyTypedInt :: Int }
4775+
4776+instance FromJSON StringlyTypedInt where
4777+ parseJSON = fmap StringlyTypedInt . parseJSON . unStringlyTypeJSON
46504778
46514779instance FromJSON NodeFSTotalStats where
46524780 parseJSON = withObject "NodeFSTotalStats" parse
tests/V5/tests.hs+46
@@ -907,7 +907,14 @@ instance Arbitrary RegexpFlag where arbitrary = sopArbitrary; shrink = genericSh
907907instance Arbitrary BoolMatch where arbitrary = sopArbitrary; shrink = genericShrink
908908instance Arbitrary Term where arbitrary = sopArbitrary; shrink = genericShrink
909909instance Arbitrary IndexSettings where arbitrary = sopArbitrary; shrink = genericShrink
910+instance Arbitrary TokenChar where arbitrary = sopArbitrary; shrink = genericShrink
911+instance Arbitrary Ngram where arbitrary = sopArbitrary; shrink = genericShrink
912+instance Arbitrary TokenizerDefinition where arbitrary = sopArbitrary; shrink = genericShrink
913+instance Arbitrary AnalyzerDefinition where arbitrary = sopArbitrary; shrink = genericShrink
914+instance Arbitrary Analysis where arbitrary = sopArbitrary; shrink = genericShrink
915+instance Arbitrary Tokenizer where arbitrary = sopArbitrary; shrink = genericShrink
910916instance Arbitrary UpdatableIndexSetting where arbitrary = sopArbitrary; shrink = genericShrink
917+instance Arbitrary Compression where arbitrary = sopArbitrary; shrink = genericShrink
911918instance Arbitrary Bytes where arbitrary = sopArbitrary; shrink = genericShrink
912919instance Arbitrary AllocationPolicy where arbitrary = sopArbitrary; shrink = genericShrink
913920instance Arbitrary InitialShardCount where arbitrary = sopArbitrary; shrink = genericShrink
@@ -1622,6 +1629,45 @@ main = hspec $ do
16221629 (IndexSettings (ShardCount 1) (ReplicaCount 0))
16231630 (NE.toList updates))
16241631
1632+ it "accepts customer analyzers" $ when' (atleast es50) $ withTestEnv $ do
1633+ _ <- deleteExampleIndex
1634+ let analysis = Analysis
1635+ (M.singleton "ex_analyzer" (AnalyzerDefinition (Just (Tokenizer "ex_tokenizer"))))
1636+ (M.singleton "ex_tokenizer"
1637+ ( TokenizerDefinitionNgram
1638+ ( Ngram 3 4 [TokenLetter,TokenDigit])
1639+ )
1640+ )
1641+ updates = [AnalysisSetting analysis]
1642+ createResp <- createIndexWith (updates ++ [NumberOfReplicas (ReplicaCount 0)]) 1 testIndex
1643+ liftIO $ validateStatus createResp 200
1644+ getResp <- getIndexSettings testIndex
1645+ liftIO $
1646+ getResp `shouldBe` Right (IndexSettingsSummary
1647+ testIndex
1648+ (IndexSettings (ShardCount 1) (ReplicaCount 0))
1649+ updates
1650+ )
1651+
1652+ it "accepts default compression codec" $ when' (atleast es50) $ withTestEnv $ do
1653+ _ <- deleteExampleIndex
1654+ let updates = [CompressionSetting CompressionDefault]
1655+ createResp <- createIndexWith (updates ++ [NumberOfReplicas (ReplicaCount 0)]) 1 testIndex
1656+ liftIO $ validateStatus createResp 200
1657+ getResp <- getIndexSettings testIndex
1658+ liftIO $ getResp `shouldBe` Right
1659+ (IndexSettingsSummary testIndex (IndexSettings (ShardCount 1) (ReplicaCount 0)) updates)
1660+
1661+ it "accepts best compression codec" $ when' (atleast es50) $ withTestEnv $ do
1662+ _ <- deleteExampleIndex
1663+ let updates = [CompressionSetting CompressionBest]
1664+ createResp <- createIndexWith (updates ++ [NumberOfReplicas (ReplicaCount 0)]) 1 testIndex
1665+ liftIO $ validateStatus createResp 200
1666+ getResp <- getIndexSettings testIndex
1667+ liftIO $ getResp `shouldBe` Right
1668+ (IndexSettingsSummary testIndex (IndexSettings (ShardCount 1) (ReplicaCount 0)) updates)
1669+
1670+
16251671 describe "Index Optimization" $ do
16261672 it "returns a successful response upon completion" $ withTestEnv $ do
16271673 _ <- createExampleIndex