296f164 · typeable/bloodhound

git.iko.soy/typeable/bloodhound

Support analyzer with custom tokenizer, only ngram for now. Also, add a function for defining index settings when the index is created since the analysis setting cannot be updated dynamically without closing the index.

296f164Andrew Martin9 years ago

src/Database/V5/Bloodhound/Client.hs+23−1

compare with master

unifiedsplit

@@ -25,6 +25,7 @@ module Database.V5.Bloodhound.Client

25 withBH

26 -- ** Indices

27 , createIndex

28+ , createIndexWith

28 , deleteIndex

29 , deleteIndex

29 , updateIndexSettings

30 , updateIndexSettings

30 , getIndexSettings

31 , getIndexSettings

@@ -512,6 +513,28 @@ createIndex indexSettings (IndexName indexName) =

512 where url = joinPath [indexName]

513 where url = joinPath [indexName]

513 body = Just $ encode indexSettings

514 body = Just $ encode indexSettings

514

515

516+-- | Create an index, providing it with any number of settings. This

517+-- is more expressive than 'createIndex' but makes is more verbose

518+-- for the common case of configuring only the shard count and

519+-- replica count.

520+createIndexWith :: MonadBH m

521+ => [UpdatableIndexSetting]

522+ -> Int -- ^ shard count

523+ -> IndexName

524+ -> m Reply

525+createIndexWith updates shards (IndexName indexName) =

526+ bindM2 put url (return (Just body))

527+ where url = joinPath [indexName]

528+ body = encode $ object

529+ ["settings" .= deepMerge

530+ ( HM.singleton "index.number_of_shards" (toJSON shards) :

531+ [u | Object u <- toJSON <$> updates]

532+ )

533+ ]

534+

535+oPath :: ToJSON a => NonEmpty Text -> a -> Value

536+oPath (k :| []) v = object [k .= v]

537+oPath (k:| (h:t)) v = object [k .= oPath (h :| t) v]

515

538

516-- | 'deleteIndex' will delete an index given a 'Server', and an 'IndexName'.

539-- | 'deleteIndex' will delete an index given a 'Server', and an 'IndexName'.

517--

540--

@@ -545,7 +568,6 @@ getIndexSettings (IndexName indexName) = do

545 parseEsResponse =<< get =<< url

568 parseEsResponse =<< get =<< url

546 where url = joinPath [indexName, "_settings"]

569 where url = joinPath [indexName, "_settings"]

547

570

548−

549-- | 'forceMergeIndex'

571-- | 'forceMergeIndex'

550--

572--

551-- The force merge API allows to force merging of one or more indices through

573-- The force merge API allows to force merging of one or more indices through

src/Database/V5/Bloodhound/Types.hs+97−1

@@ -202,6 +202,7 @@ module Database.V5.Bloodhound.Types

202 , ZeroTermsQuery(..)

203 , CutoffFrequency(..)

204 , Analyzer(..)

205+ , Tokenizer(..)

205 , MaxExpansions(..)

206 , MaxExpansions(..)

206 , Lenient(..)

207 , Lenient(..)

207 , MatchQueryType(..)

208 , MatchQueryType(..)

@@ -376,6 +377,12 @@ module Database.V5.Bloodhound.Types

376

377

377 , EsUsername(..)

378 , EsUsername(..)

378 , EsPassword(..)

379 , EsPassword(..)

380+

381+ , Analysis(..)

382+ , AnalyzerDefinition(..)

383+ , TokenizerDefinition(..)

384+ , Ngram(..)

385+ , TokenChar(..)

379 ) where

386 ) where

380

387

381import Control.Applicative as A

388import Control.Applicative as A

@@ -583,8 +590,89 @@ data UpdatableIndexSetting = NumberOfReplicas ReplicaCount

583 | IndexCompoundOnFlush Bool

590 | IndexCompoundOnFlush Bool

584 | WarmerEnabled Bool

591 | WarmerEnabled Bool

585 | MappingTotalFieldsLimit Int

592 | MappingTotalFieldsLimit Int

593+ | AnalysisSetting Analysis

594+ -- ^ Analysis is not a dynamic setting and can only be performed on a closed index.

586 deriving (Eq, Show, Generic, Typeable)

595 deriving (Eq, Show, Generic, Typeable)

587

596

597+data Analysis = Analysis

598+ { analysisAnalyzer :: M.Map Text AnalyzerDefinition

599+ , analysisTokenizer :: M.Map Text TokenizerDefinition

600+ } deriving (Eq,Show,Generic,Typeable)

601+

602+instance ToJSON Analysis where

603+ toJSON (Analysis analyzer tokenizer) = object

604+ [ "analyzer" .= analyzer

605+ , "tokenizer" .= tokenizer

606+ ]

607+

608+instance FromJSON Analysis where

609+ parseJSON = withObject "Analysis" $ \m -> Analysis

610+ <$> m .: "analyzer"

611+ <*> m .: "tokenizer"

612+

613+data AnalyzerDefinition = AnalyzerDefinition

614+ { analyzerDefinitionTokenizer :: Maybe Tokenizer

615+ } deriving (Eq,Show,Generic,Typeable)

616+

617+instance ToJSON AnalyzerDefinition where

618+ toJSON (AnalyzerDefinition tokenizer) = object $ catMaybes

619+ [ fmap ("tokenizer" .=) tokenizer

620+ ]

621+

622+instance FromJSON AnalyzerDefinition where

623+ parseJSON = withObject "AnalyzerDefinition" $ \m -> AnalyzerDefinition

624+ <$> m .:? "tokenizer"

625+

626+

627+data TokenizerDefinition

628+ = TokenizerDefinitionNgram Ngram

629+ deriving (Eq,Show,Generic,Typeable)

630+

631+instance ToJSON TokenizerDefinition where

632+ toJSON x = case x of

633+ TokenizerDefinitionNgram (Ngram minGram maxGram tokenChars) -> object

634+ [ "type" .= ("ngram" :: Text)

635+ , "min_gram" .= minGram

636+ , "max_gram" .= maxGram

637+ , "token_chars" .= tokenChars

638+ ]

639+

640+instance FromJSON TokenizerDefinition where

641+ parseJSON = withObject "TokenizerDefinition" $ \m -> do

642+ typ <- m .: "type" :: Parser Text

643+ case typ of

644+ "ngram" -> fmap TokenizerDefinitionNgram $ Ngram

645+ <$> (fmap unStringlyTypedInt (m .: "min_gram"))

646+ <*> (fmap unStringlyTypedInt (m .: "max_gram"))

647+ <*> m .: "token_chars"

648+ _ -> fail "invalid TokenizerDefinition"

649+

650+data Ngram = Ngram

651+ { ngramMinGram :: Int

652+ , ngramMaxGram :: Int

653+ , ngramTokenChars :: [TokenChar]

654+ } deriving (Eq,Show,Generic,Typeable)

655+

656+data TokenChar = TokenLetter | TokenDigit | TokenWhitespace | TokenPunctuation | TokenSymbol

657+ deriving (Eq,Read,Show,Generic,Typeable)

658+

659+instance ToJSON TokenChar where

660+ toJSON t = String $ case t of

661+ TokenLetter -> "letter"

662+ TokenDigit -> "digit"

663+ TokenWhitespace -> "whitespace"

664+ TokenPunctuation -> "punctuation"

665+ TokenSymbol -> "symbol"

666+

667+instance FromJSON TokenChar where

668+ parseJSON = withText "TokenChar" $ \t -> case t of

669+ "letter" -> return TokenLetter

670+ "digit" -> return TokenDigit

671+ "whitespace" -> return TokenWhitespace

672+ "punctuation" -> return TokenPunctuation

673+ "symbol" -> return TokenSymbol

674+ _ -> fail "invalid TokenChar"

675+

588data AllocationPolicy = AllocAll

676data AllocationPolicy = AllocAll

589 -- ^ Allows shard allocation for all shards.

677 -- ^ Allows shard allocation for all shards.

590 | AllocPrimaries

678 | AllocPrimaries

@@ -1020,6 +1108,8 @@ newtype CutoffFrequency =

1020 CutoffFrequency Double deriving (Eq, Read, Show, Generic, ToJSON, FromJSON, Typeable)

1108 CutoffFrequency Double deriving (Eq, Read, Show, Generic, ToJSON, FromJSON, Typeable)

1021newtype Analyzer =

1109newtype Analyzer =

1022 Analyzer Text deriving (Eq, Read, Show, Generic, ToJSON, FromJSON, Typeable)

1110 Analyzer Text deriving (Eq, Read, Show, Generic, ToJSON, FromJSON, Typeable)

1111+newtype Tokenizer =

1112+ Tokenizer Text deriving (Eq, Read, Show, Generic, ToJSON, FromJSON, Typeable)

1023newtype MaxExpansions =

1113newtype MaxExpansions =

1024 MaxExpansions Int deriving (Eq, Read, Show, Generic, ToJSON, FromJSON, Typeable)

1114 MaxExpansions Int deriving (Eq, Read, Show, Generic, ToJSON, FromJSON, Typeable)

1025

1115

@@ -2990,6 +3080,7 @@ instance ToJSON UpdatableIndexSetting where

2990 toJSON (BlocksWrite x) = oPath ("blocks" :| ["write"]) x

3080 toJSON (BlocksWrite x) = oPath ("blocks" :| ["write"]) x

2991 toJSON (BlocksMetaData x) = oPath ("blocks" :| ["metadata"]) x

3081 toJSON (BlocksMetaData x) = oPath ("blocks" :| ["metadata"]) x

2992 toJSON (MappingTotalFieldsLimit x) = oPath ("index" :| ["mapping","total_fields","limit"]) x

3082 toJSON (MappingTotalFieldsLimit x) = oPath ("index" :| ["mapping","total_fields","limit"]) x

3083+ toJSON (AnalysisSetting x) = oPath ("index" :| ["analysis"]) x

2993

3084

2994instance FromJSON UpdatableIndexSetting where

3085instance FromJSON UpdatableIndexSetting where

2995 parseJSON = withObject "UpdatableIndexSetting" parse

3086 parseJSON = withObject "UpdatableIndexSetting" parse

@@ -3022,6 +3113,7 @@ instance FromJSON UpdatableIndexSetting where

3022 <|> blocksWrite `taggedAt` ["blocks", "write"]

3113 <|> blocksWrite `taggedAt` ["blocks", "write"]

3023 <|> blocksMetaData `taggedAt` ["blocks", "metadata"]

3114 <|> blocksMetaData `taggedAt` ["blocks", "metadata"]

3024 <|> mappingTotalFieldsLimit `taggedAt` ["index", "mapping", "total_fields", "limit"]

3115 <|> mappingTotalFieldsLimit `taggedAt` ["index", "mapping", "total_fields", "limit"]

3116+ <|> analysisSetting `taggedAt` ["index", "analysis"]

3025 where taggedAt f ks = taggedAt' f (Object o) ks

3117 where taggedAt f ks = taggedAt' f (Object o) ks

3026 taggedAt' f v [] = f =<< (parseJSON v <|> (parseJSON (unStringlyTypeJSON v)))

3118 taggedAt' f v [] = f =<< (parseJSON v <|> (parseJSON (unStringlyTypeJSON v)))

3027 taggedAt' f v (k:ks) = withObject "Object" (\o -> do v' <- o .: k

3119 taggedAt' f v (k:ks) = withObject "Object" (\o -> do v' <- o .: k

@@ -3055,6 +3147,7 @@ instance FromJSON UpdatableIndexSetting where

3055 blocksWrite = pure . BlocksWrite

3147 blocksWrite = pure . BlocksWrite

3056 blocksMetaData = pure . BlocksMetaData

3148 blocksMetaData = pure . BlocksMetaData

3057 mappingTotalFieldsLimit = pure . MappingTotalFieldsLimit

3149 mappingTotalFieldsLimit = pure . MappingTotalFieldsLimit

3150+ analysisSetting = pure . AnalysisSetting

3058

3151

3059instance FromJSON IndexSettingsSummary where

3152instance FromJSON IndexSettingsSummary where

3060 parseJSON = withObject "IndexSettingsSummary" parse

3153 parseJSON = withObject "IndexSettingsSummary" parse

@@ -4643,10 +4736,13 @@ instance FromJSON NodeDataPathStats where

4643

4736

4644newtype StringlyTypedDouble = StringlyTypedDouble { unStringlyTypedDouble :: Double }

4737newtype StringlyTypedDouble = StringlyTypedDouble { unStringlyTypedDouble :: Double }

4645

4738

4646−

4647instance FromJSON StringlyTypedDouble where

4739instance FromJSON StringlyTypedDouble where

4648 parseJSON = fmap StringlyTypedDouble . parseJSON . unStringlyTypeJSON

4740 parseJSON = fmap StringlyTypedDouble . parseJSON . unStringlyTypeJSON

4649

4741

4742+newtype StringlyTypedInt = StringlyTypedInt { unStringlyTypedInt :: Int }

4743+

4744+instance FromJSON StringlyTypedInt where

4745+ parseJSON = fmap StringlyTypedInt . parseJSON . unStringlyTypeJSON

4650

4746

4651instance FromJSON NodeFSTotalStats where

4747instance FromJSON NodeFSTotalStats where

4652 parseJSON = withObject "NodeFSTotalStats" parse

4748 parseJSON = withObject "NodeFSTotalStats" parse

tests/V5/tests.hs+26

@@ -907,6 +907,12 @@ instance Arbitrary RegexpFlag where arbitrary = sopArbitrary; shrink = genericSh

907instance Arbitrary BoolMatch where arbitrary = sopArbitrary; shrink = genericShrink

908instance Arbitrary Term where arbitrary = sopArbitrary; shrink = genericShrink

909instance Arbitrary IndexSettings where arbitrary = sopArbitrary; shrink = genericShrink

910+instance Arbitrary TokenChar where arbitrary = sopArbitrary; shrink = genericShrink

911+instance Arbitrary Ngram where arbitrary = sopArbitrary; shrink = genericShrink

912+instance Arbitrary TokenizerDefinition where arbitrary = sopArbitrary; shrink = genericShrink

913+instance Arbitrary AnalyzerDefinition where arbitrary = sopArbitrary; shrink = genericShrink

914+instance Arbitrary Analysis where arbitrary = sopArbitrary; shrink = genericShrink

915+instance Arbitrary Tokenizer where arbitrary = sopArbitrary; shrink = genericShrink

910instance Arbitrary UpdatableIndexSetting where arbitrary = sopArbitrary; shrink = genericShrink

916instance Arbitrary UpdatableIndexSetting where arbitrary = sopArbitrary; shrink = genericShrink

911instance Arbitrary Bytes where arbitrary = sopArbitrary; shrink = genericShrink

917instance Arbitrary Bytes where arbitrary = sopArbitrary; shrink = genericShrink

912instance Arbitrary AllocationPolicy where arbitrary = sopArbitrary; shrink = genericShrink

918instance Arbitrary AllocationPolicy where arbitrary = sopArbitrary; shrink = genericShrink

@@ -1622,6 +1628,26 @@ main = hspec $ do

1622 (IndexSettings (ShardCount 1) (ReplicaCount 0))

1628 (IndexSettings (ShardCount 1) (ReplicaCount 0))

1623 (NE.toList updates))

1629 (NE.toList updates))

1624

1630

1631+ it "accepts customer analyzers" $ when' (atleast es50) $ withTestEnv $ do

1632+ _ <- deleteExampleIndex

1633+ let analysis = Analysis

1634+ (M.singleton "ex_analyzer" (AnalyzerDefinition (Just (Tokenizer "ex_tokenizer"))))

1635+ (M.singleton "ex_tokenizer"

1636+ ( TokenizerDefinitionNgram

1637+ ( Ngram 3 4 [TokenLetter,TokenDigit])

1638+ )

1639+ )

1640+ updates = [AnalysisSetting analysis]

1641+ createResp <- createIndexWith (updates ++ [NumberOfReplicas (ReplicaCount 0)]) 1 testIndex

1642+ liftIO $ validateStatus createResp 200

1643+ getResp <- getIndexSettings testIndex

1644+ liftIO $

1645+ getResp `shouldBe` Right (IndexSettingsSummary

1646+ testIndex

1647+ (IndexSettings (ShardCount 1) (ReplicaCount 0))

1648+ updates

1649+ )

1650+

1625 describe "Index Optimization" $ do

1651 describe "Index Optimization" $ do

1626 it "returns a successful response upon completion" $ withTestEnv $ do

1652 it "returns a successful response upon completion" $ withTestEnv $ do

1627 _ <- createExampleIndex

1653 _ <- createExampleIndex