git.iko.soy/typeable/bloodhound
on master help sign in
Support analyzer with custom tokenizer, only ngram for now. Also, add a function for defining index settings when the index is created since the analysis setting cannot be updated dynamically without closing the index.
commit296f164df53980fcafa0771c9c58fde95a447c86 authorAndrew Martin <andrew.thaddeus@gmail.com> dateTue 10 Oct 2017, 09:24 (9 years ago) parentf183995
3 files changed+146−2
src/Database/V5/Bloodhound/Client.hs+23−1 src/Database/V5/Bloodhound/Types.hs+97−1 tests/V5/tests.hs+26
src/Database/V5/Bloodhound/Client.hs+23−1
@@ -25,6 +25,7 @@ module Database.V5.Bloodhound.Client
2525 withBH
2626 -- ** Indices
2727 , createIndex
28+ , createIndexWith
2829 , deleteIndex
2930 , updateIndexSettings
3031 , getIndexSettings
@@ -512,6 +513,28 @@ createIndex indexSettings (IndexName indexName) =
512513 where url = joinPath [indexName]
513514 body = Just $ encode indexSettings
514515
516+-- | Create an index, providing it with any number of settings. This
517+-- is more expressive than 'createIndex' but makes is more verbose
518+-- for the common case of configuring only the shard count and
519+-- replica count.
520+createIndexWith :: MonadBH m
521+ => [UpdatableIndexSetting]
522+ -> Int -- ^ shard count
523+ -> IndexName
524+ -> m Reply
525+createIndexWith updates shards (IndexName indexName) =
526+ bindM2 put url (return (Just body))
527+ where url = joinPath [indexName]
528+ body = encode $ object
529+ ["settings" .= deepMerge
530+ ( HM.singleton "index.number_of_shards" (toJSON shards) :
531+ [u | Object u <- toJSON <$> updates]
532+ )
533+ ]
534+
535+oPath :: ToJSON a => NonEmpty Text -> a -> Value
536+oPath (k :| []) v = object [k .= v]
537+oPath (k:| (h:t)) v = object [k .= oPath (h :| t) v]
515538
516539-- | 'deleteIndex' will delete an index given a 'Server', and an 'IndexName'.
517540--
@@ -545,7 +568,6 @@ getIndexSettings (IndexName indexName) = do
545568 parseEsResponse =<< get =<< url
546569 where url = joinPath [indexName, "_settings"]
547570
548−
549571-- | 'forceMergeIndex'
550572--
551573-- The force merge API allows to force merging of one or more indices through
src/Database/V5/Bloodhound/Types.hs+97−1
@@ -202,6 +202,7 @@ module Database.V5.Bloodhound.Types
202202 , ZeroTermsQuery(..)
203203 , CutoffFrequency(..)
204204 , Analyzer(..)
205+ , Tokenizer(..)
205206 , MaxExpansions(..)
206207 , Lenient(..)
207208 , MatchQueryType(..)
@@ -376,6 +377,12 @@ module Database.V5.Bloodhound.Types
376377
377378 , EsUsername(..)
378379 , EsPassword(..)
380+
381+ , Analysis(..)
382+ , AnalyzerDefinition(..)
383+ , TokenizerDefinition(..)
384+ , Ngram(..)
385+ , TokenChar(..)
379386 ) where
380387
381388import Control.Applicative as A
@@ -583,8 +590,89 @@ data UpdatableIndexSetting = NumberOfReplicas ReplicaCount
583590 | IndexCompoundOnFlush Bool
584591 | WarmerEnabled Bool
585592 | MappingTotalFieldsLimit Int
593+ | AnalysisSetting Analysis
594+ -- ^ Analysis is not a dynamic setting and can only be performed on a closed index.
586595 deriving (Eq, Show, Generic, Typeable)
587596
597+data Analysis = Analysis
598+ { analysisAnalyzer :: M.Map Text AnalyzerDefinition
599+ , analysisTokenizer :: M.Map Text TokenizerDefinition
600+ } deriving (Eq,Show,Generic,Typeable)
601+
602+instance ToJSON Analysis where
603+ toJSON (Analysis analyzer tokenizer) = object
604+ [ "analyzer" .= analyzer
605+ , "tokenizer" .= tokenizer
606+ ]
607+
608+instance FromJSON Analysis where
609+ parseJSON = withObject "Analysis" $ \m -> Analysis
610+ <$> m .: "analyzer"
611+ <*> m .: "tokenizer"
612+
613+data AnalyzerDefinition = AnalyzerDefinition
614+ { analyzerDefinitionTokenizer :: Maybe Tokenizer
615+ } deriving (Eq,Show,Generic,Typeable)
616+
617+instance ToJSON AnalyzerDefinition where
618+ toJSON (AnalyzerDefinition tokenizer) = object $ catMaybes
619+ [ fmap ("tokenizer" .=) tokenizer
620+ ]
621+
622+instance FromJSON AnalyzerDefinition where
623+ parseJSON = withObject "AnalyzerDefinition" $ \m -> AnalyzerDefinition
624+ <$> m .:? "tokenizer"
625+
626+
627+data TokenizerDefinition
628+ = TokenizerDefinitionNgram Ngram
629+ deriving (Eq,Show,Generic,Typeable)
630+
631+instance ToJSON TokenizerDefinition where
632+ toJSON x = case x of
633+ TokenizerDefinitionNgram (Ngram minGram maxGram tokenChars) -> object
634+ [ "type" .= ("ngram" :: Text)
635+ , "min_gram" .= minGram
636+ , "max_gram" .= maxGram
637+ , "token_chars" .= tokenChars
638+ ]
639+
640+instance FromJSON TokenizerDefinition where
641+ parseJSON = withObject "TokenizerDefinition" $ \m -> do
642+ typ <- m .: "type" :: Parser Text
643+ case typ of
644+ "ngram" -> fmap TokenizerDefinitionNgram $ Ngram
645+ <$> (fmap unStringlyTypedInt (m .: "min_gram"))
646+ <*> (fmap unStringlyTypedInt (m .: "max_gram"))
647+ <*> m .: "token_chars"
648+ _ -> fail "invalid TokenizerDefinition"
649+
650+data Ngram = Ngram
651+ { ngramMinGram :: Int
652+ , ngramMaxGram :: Int
653+ , ngramTokenChars :: [TokenChar]
654+ } deriving (Eq,Show,Generic,Typeable)
655+
656+data TokenChar = TokenLetter | TokenDigit | TokenWhitespace | TokenPunctuation | TokenSymbol
657+ deriving (Eq,Read,Show,Generic,Typeable)
658+
659+instance ToJSON TokenChar where
660+ toJSON t = String $ case t of
661+ TokenLetter -> "letter"
662+ TokenDigit -> "digit"
663+ TokenWhitespace -> "whitespace"
664+ TokenPunctuation -> "punctuation"
665+ TokenSymbol -> "symbol"
666+
667+instance FromJSON TokenChar where
668+ parseJSON = withText "TokenChar" $ \t -> case t of
669+ "letter" -> return TokenLetter
670+ "digit" -> return TokenDigit
671+ "whitespace" -> return TokenWhitespace
672+ "punctuation" -> return TokenPunctuation
673+ "symbol" -> return TokenSymbol
674+ _ -> fail "invalid TokenChar"
675+
588676data AllocationPolicy = AllocAll
589677 -- ^ Allows shard allocation for all shards.
590678 | AllocPrimaries
@@ -1020,6 +1108,8 @@ newtype CutoffFrequency =
10201108 CutoffFrequency Double deriving (Eq, Read, Show, Generic, ToJSON, FromJSON, Typeable)
10211109newtype Analyzer =
10221110 Analyzer Text deriving (Eq, Read, Show, Generic, ToJSON, FromJSON, Typeable)
1111+newtype Tokenizer =
1112+ Tokenizer Text deriving (Eq, Read, Show, Generic, ToJSON, FromJSON, Typeable)
10231113newtype MaxExpansions =
10241114 MaxExpansions Int deriving (Eq, Read, Show, Generic, ToJSON, FromJSON, Typeable)
10251115
@@ -2990,6 +3080,7 @@ instance ToJSON UpdatableIndexSetting where
29903080 toJSON (BlocksWrite x) = oPath ("blocks" :| ["write"]) x
29913081 toJSON (BlocksMetaData x) = oPath ("blocks" :| ["metadata"]) x
29923082 toJSON (MappingTotalFieldsLimit x) = oPath ("index" :| ["mapping","total_fields","limit"]) x
3083+ toJSON (AnalysisSetting x) = oPath ("index" :| ["analysis"]) x
29933084
29943085instance FromJSON UpdatableIndexSetting where
29953086 parseJSON = withObject "UpdatableIndexSetting" parse
@@ -3022,6 +3113,7 @@ instance FromJSON UpdatableIndexSetting where
30223113 <|> blocksWrite `taggedAt` ["blocks", "write"]
30233114 <|> blocksMetaData `taggedAt` ["blocks", "metadata"]
30243115 <|> mappingTotalFieldsLimit `taggedAt` ["index", "mapping", "total_fields", "limit"]
3116+ <|> analysisSetting `taggedAt` ["index", "analysis"]
30253117 where taggedAt f ks = taggedAt' f (Object o) ks
30263118 taggedAt' f v [] = f =<< (parseJSON v <|> (parseJSON (unStringlyTypeJSON v)))
30273119 taggedAt' f v (k:ks) = withObject "Object" (\o -> do v' <- o .: k
@@ -3055,6 +3147,7 @@ instance FromJSON UpdatableIndexSetting where
30553147 blocksWrite = pure . BlocksWrite
30563148 blocksMetaData = pure . BlocksMetaData
30573149 mappingTotalFieldsLimit = pure . MappingTotalFieldsLimit
3150+ analysisSetting = pure . AnalysisSetting
30583151
30593152instance FromJSON IndexSettingsSummary where
30603153 parseJSON = withObject "IndexSettingsSummary" parse
@@ -4643,10 +4736,13 @@ instance FromJSON NodeDataPathStats where
46434736
46444737newtype StringlyTypedDouble = StringlyTypedDouble { unStringlyTypedDouble :: Double }
46454738
4646−
46474739instance FromJSON StringlyTypedDouble where
46484740 parseJSON = fmap StringlyTypedDouble . parseJSON . unStringlyTypeJSON
46494741
4742+newtype StringlyTypedInt = StringlyTypedInt { unStringlyTypedInt :: Int }
4743+
4744+instance FromJSON StringlyTypedInt where
4745+ parseJSON = fmap StringlyTypedInt . parseJSON . unStringlyTypeJSON
46504746
46514747instance FromJSON NodeFSTotalStats where
46524748 parseJSON = withObject "NodeFSTotalStats" parse
tests/V5/tests.hs+26
@@ -907,6 +907,12 @@ instance Arbitrary RegexpFlag where arbitrary = sopArbitrary; shrink = genericSh
907907instance Arbitrary BoolMatch where arbitrary = sopArbitrary; shrink = genericShrink
908908instance Arbitrary Term where arbitrary = sopArbitrary; shrink = genericShrink
909909instance Arbitrary IndexSettings where arbitrary = sopArbitrary; shrink = genericShrink
910+instance Arbitrary TokenChar where arbitrary = sopArbitrary; shrink = genericShrink
911+instance Arbitrary Ngram where arbitrary = sopArbitrary; shrink = genericShrink
912+instance Arbitrary TokenizerDefinition where arbitrary = sopArbitrary; shrink = genericShrink
913+instance Arbitrary AnalyzerDefinition where arbitrary = sopArbitrary; shrink = genericShrink
914+instance Arbitrary Analysis where arbitrary = sopArbitrary; shrink = genericShrink
915+instance Arbitrary Tokenizer where arbitrary = sopArbitrary; shrink = genericShrink
910916instance Arbitrary UpdatableIndexSetting where arbitrary = sopArbitrary; shrink = genericShrink
911917instance Arbitrary Bytes where arbitrary = sopArbitrary; shrink = genericShrink
912918instance Arbitrary AllocationPolicy where arbitrary = sopArbitrary; shrink = genericShrink
@@ -1622,6 +1628,26 @@ main = hspec $ do
16221628 (IndexSettings (ShardCount 1) (ReplicaCount 0))
16231629 (NE.toList updates))
16241630
1631+ it "accepts customer analyzers" $ when' (atleast es50) $ withTestEnv $ do
1632+ _ <- deleteExampleIndex
1633+ let analysis = Analysis
1634+ (M.singleton "ex_analyzer" (AnalyzerDefinition (Just (Tokenizer "ex_tokenizer"))))
1635+ (M.singleton "ex_tokenizer"
1636+ ( TokenizerDefinitionNgram
1637+ ( Ngram 3 4 [TokenLetter,TokenDigit])
1638+ )
1639+ )
1640+ updates = [AnalysisSetting analysis]
1641+ createResp <- createIndexWith (updates ++ [NumberOfReplicas (ReplicaCount 0)]) 1 testIndex
1642+ liftIO $ validateStatus createResp 200
1643+ getResp <- getIndexSettings testIndex
1644+ liftIO $
1645+ getResp `shouldBe` Right (IndexSettingsSummary
1646+ testIndex
1647+ (IndexSettings (ShardCount 1) (ReplicaCount 0))
1648+ updates
1649+ )
1650+
16251651 describe "Index Optimization" $ do
16261652 it "returns a successful response upon completion" $ withTestEnv $ do
16271653 _ <- createExampleIndex