7fce32a · typeable/bloodhound

git.iko.soy/typeable/bloodhound

on master help sign in

Add support for token filters. The supported filters

7fce32aAndrew Martin9 years ago

src/Database/V5/Bloodhound/Types.hs+226−3

compare with master

unifiedsplit

@@ -203,6 +203,7 @@ module Database.V5.Bloodhound.Types

203 , CutoffFrequency(..)

204 , Analyzer(..)

205 , Tokenizer(..)

206+ , TokenFilter(..)

206 , MaxExpansions(..)

207 , MaxExpansions(..)

207 , Lenient(..)

208 , Lenient(..)

208 , MatchQueryType(..)

209 , MatchQueryType(..)

@@ -381,8 +382,11 @@ module Database.V5.Bloodhound.Types

381 , Analysis(..)

382 , Analysis(..)

382 , AnalyzerDefinition(..)

383 , AnalyzerDefinition(..)

383 , TokenizerDefinition(..)

384 , TokenizerDefinition(..)

385+ , TokenFilterDefinition(..)

384 , Ngram(..)

386 , Ngram(..)

385 , TokenChar(..)

387 , TokenChar(..)

388+ , Shingle(..)

389+ , Language(..)

386 ) where

390 ) where

387

391

388import Control.Applicative as A

392import Control.Applicative as A

@@ -597,32 +601,244 @@ data UpdatableIndexSetting = NumberOfReplicas ReplicaCount

597data Analysis = Analysis

601data Analysis = Analysis

598 { analysisAnalyzer :: M.Map Text AnalyzerDefinition

602 { analysisAnalyzer :: M.Map Text AnalyzerDefinition

599 , analysisTokenizer :: M.Map Text TokenizerDefinition

603 , analysisTokenizer :: M.Map Text TokenizerDefinition

604+ , analysisTokenFilter :: M.Map Text TokenFilterDefinition

600 } deriving (Eq,Show,Generic,Typeable)

605 } deriving (Eq,Show,Generic,Typeable)

601

606

602instance ToJSON Analysis where

607instance ToJSON Analysis where

603− toJSON (Analysis analyzer tokenizer) = object

608+ toJSON (Analysis analyzer tokenizer tokenFilter) = object

604 [ "analyzer" .= analyzer

609 [ "analyzer" .= analyzer

605 , "tokenizer" .= tokenizer

610 , "tokenizer" .= tokenizer

611+ , "filter" .= tokenFilter

606 ]

612 ]

607

613

608instance FromJSON Analysis where

614instance FromJSON Analysis where

609 parseJSON = withObject "Analysis" $ \m -> Analysis

615 parseJSON = withObject "Analysis" $ \m -> Analysis

610 <$> m .: "analyzer"

616 <$> m .: "analyzer"

611− <*> m .: "tokenizer"

617+ <*> m .:? "tokenizer" .!= M.empty

618+ <*> m .:? "filter" .!= M.empty

612

619

613data AnalyzerDefinition = AnalyzerDefinition

620data AnalyzerDefinition = AnalyzerDefinition

614 { analyzerDefinitionTokenizer :: Maybe Tokenizer

621 { analyzerDefinitionTokenizer :: Maybe Tokenizer

622+ , analyzerDefinitionFilter :: [TokenFilter]

615 } deriving (Eq,Show,Generic,Typeable)

623 } deriving (Eq,Show,Generic,Typeable)

616

624

617instance ToJSON AnalyzerDefinition where

625instance ToJSON AnalyzerDefinition where

618− toJSON (AnalyzerDefinition tokenizer) = object $ catMaybes

626+ toJSON (AnalyzerDefinition tokenizer tokenFilter) = object $ catMaybes

619 [ fmap ("tokenizer" .=) tokenizer

627 [ fmap ("tokenizer" .=) tokenizer

628+ , Just $ "filter" .= tokenFilter

620 ]

629 ]

621

630

622instance FromJSON AnalyzerDefinition where

631instance FromJSON AnalyzerDefinition where

623 parseJSON = withObject "AnalyzerDefinition" $ \m -> AnalyzerDefinition

632 parseJSON = withObject "AnalyzerDefinition" $ \m -> AnalyzerDefinition

624 <$> m .:? "tokenizer"

633 <$> m .:? "tokenizer"

634+ <*> m .:? "filter" .!= []

625

635

636+-- | Token filters are used to create custom analyzers.

637+data TokenFilterDefinition

638+ = TokenFilterDefinitionLowercase Language

639+ | TokenFilterDefinitionUppercase Language

640+ | TokenFilterDefinitionApostrophe

641+ | TokenFilterDefinitionReverse

642+ | TokenFilterDefinitionSnowball Language

643+ | TokenFilterDefinitionShingle Shingle

644+ deriving (Eq,Show,Generic)

645+

646+instance ToJSON TokenFilterDefinition where

647+ toJSON x = case x of

648+ TokenFilterDefinitionLowercase lang -> object

649+ [ "type" .= ("lowercase" :: Text)

650+ , "language" .= languageToText lang

651+ ]

652+ TokenFilterDefinitionUppercase lang -> object

653+ [ "type" .= ("uppercase" :: Text)

654+ , "language" .= languageToText lang

655+ ]

656+ TokenFilterDefinitionApostrophe -> object

657+ [ "type" .= ("apostrophe" :: Text)

658+ ]

659+ TokenFilterDefinitionReverse -> object

660+ [ "type" .= ("reverse" :: Text)

661+ ]

662+ TokenFilterDefinitionSnowball lang -> object

663+ [ "type" .= ("snowball" :: Text)

664+ , "language" .= languageToText lang

665+ ]

666+ TokenFilterDefinitionShingle s -> object

667+ [ "type" .= ("shingle" :: Text)

668+ , "max_shingle_size" .= shingleMaxSize s

669+ , "min_shingle_size" .= shingleMinSize s

670+ , "output_unigrams" .= shingleOutputUnigrams s

671+ , "output_unigrams_if_no_shingles" .= shingleOutputUnigramsIfNoShingles s

672+ , "token_separator" .= shingleTokenSeparator s

673+ , "filler_token" .= shingleFillerToken s

674+ ]

675+

676+instance FromJSON TokenFilterDefinition where

677+ parseJSON = withObject "TokenFilterDefinition" $ \m -> do

678+ t <- m .: "type"

679+ case (t :: Text) of

680+ "reverse" -> return TokenFilterDefinitionReverse

681+ "apostrophe" -> return TokenFilterDefinitionApostrophe

682+ "lowercase" -> TokenFilterDefinitionLowercase

683+ <$> m .:? "language" .!= English

684+ "uppercase" -> TokenFilterDefinitionUppercase

685+ <$> m .:? "language" .!= English

686+ "snowball" -> TokenFilterDefinitionSnowball

687+ <$> m .:? "language" .!= English

688+ "shingle" -> fmap TokenFilterDefinitionShingle $ Shingle

689+ <$> (fmap.fmap) unStringlyTypedInt (m .:? "max_shingle_size") .!= 2

690+ <*> (fmap.fmap) unStringlyTypedInt (m .:? "min_shingle_size") .!= 2

691+ <*> (fmap.fmap) unStringlyTypedBool (m .:? "output_unigrams") .!= True

692+ <*> (fmap.fmap) unStringlyTypedBool (m .:? "output_unigrams_if_no_shingles") .!= False

693+ <*> m .:? "token_separator" .!= " "

694+ <*> m .:? "filler_token" .!= "_"

695+ _ -> fail ("unrecognized token filter type: " ++ T.unpack t)

696+

697+-- | The set of languages that can be passed to various analyzers,

698+-- filters, etc. in ElasticSearch. Most data types in this module

699+-- that have a 'Language' field are actually only actually to

700+-- handle a subset of these languages. Consult the official

701+-- ElasticSearch documentation to see what is actually supported.

702+data Language

703+ = Arabic

704+ | Armenian

705+ | Basque

706+ | Bengali

707+ | Brazilian

708+ | Bulgarian

709+ | Catalan

710+ | Cjk

711+ | Czech

712+ | Danish

713+ | Dutch

714+ | English

715+ | Finnish

716+ | French

717+ | Galician

718+ | German

719+ | German2

720+ | Greek

721+ | Hindi

722+ | Hungarian

723+ | Indonesian

724+ | Irish

725+ | Italian

726+ | Kp

727+ | Latvian

728+ | Lithuanian

729+ | Lovins

730+ | Norwegian

731+ | Persian

732+ | Porter

733+ | Portuguese

734+ | Romanian

735+ | Russian

736+ | Sorani

737+ | Spanish

738+ | Swedish

739+ | Thai

740+ | Turkish

741+ deriving (Show,Eq,Generic)

742+

743+instance ToJSON Language where

744+ toJSON = Data.Aeson.String . languageToText

745+

746+instance FromJSON Language where

747+ parseJSON = withText "Language" $ \t -> case languageFromText t of

748+ Nothing -> fail "not a supported ElasticSearch language"

749+ Just lang -> return lang

750+

751+languageToText :: Language -> Text

752+languageToText x = case x of

753+ Arabic -> "arabic"

754+ Armenian -> "armenian"

755+ Basque -> "basque"

756+ Bengali -> "bengali"

757+ Brazilian -> "brazilian"

758+ Bulgarian -> "bulgarian"

759+ Catalan -> "catalan"

760+ Cjk -> "cjk"

761+ Czech -> "czech"

762+ Danish -> "danish"

763+ Dutch -> "dutch"

764+ English -> "english"

765+ Finnish -> "finnish"

766+ French -> "french"

767+ Galician -> "galician"

768+ German -> "german"

769+ German2 -> "german2"

770+ Greek -> "greek"

771+ Hindi -> "hindi"

772+ Hungarian -> "hungarian"

773+ Indonesian -> "indonesian"

774+ Irish -> "irish"

775+ Italian -> "italian"

776+ Kp -> "kp"

777+ Latvian -> "latvian"

778+ Lithuanian -> "lithuanian"

779+ Lovins -> "lovins"

780+ Norwegian -> "norwegian"

781+ Persian -> "persian"

782+ Porter -> "porter"

783+ Portuguese -> "portuguese"

784+ Romanian -> "romanian"

785+ Russian -> "russian"

786+ Sorani -> "sorani"

787+ Spanish -> "spanish"

788+ Swedish -> "swedish"

789+ Thai -> "thai"

790+ Turkish -> "turkish"

791+

792+languageFromText :: Text -> Maybe Language

793+languageFromText x = case x of

794+ "arabic" -> Just Arabic

795+ "armenian" -> Just Armenian

796+ "basque" -> Just Basque

797+ "bengali" -> Just Bengali

798+ "brazilian" -> Just Brazilian

799+ "bulgarian" -> Just Bulgarian

800+ "catalan" -> Just Catalan

801+ "cjk" -> Just Cjk

802+ "czech" -> Just Czech

803+ "danish" -> Just Danish

804+ "dutch" -> Just Dutch

805+ "english" -> Just English

806+ "finnish" -> Just Finnish

807+ "french" -> Just French

808+ "galician" -> Just Galician

809+ "german" -> Just German

810+ "german2" -> Just German2

811+ "greek" -> Just Greek

812+ "hindi" -> Just Hindi

813+ "hungarian" -> Just Hungarian

814+ "indonesian" -> Just Indonesian

815+ "irish" -> Just Irish

816+ "italian" -> Just Italian

817+ "kp" -> Just Kp

818+ "latvian" -> Just Latvian

819+ "lithuanian" -> Just Lithuanian

820+ "lovins" -> Just Lovins

821+ "norwegian" -> Just Norwegian

822+ "persian" -> Just Persian

823+ "porter" -> Just Porter

824+ "portuguese" -> Just Portuguese

825+ "romanian" -> Just Romanian

826+ "russian" -> Just Russian

827+ "sorani" -> Just Sorani

828+ "spanish" -> Just Spanish

829+ "swedish" -> Just Swedish

830+ "thai" -> Just Thai

831+ "turkish" -> Just Turkish

832+ _ -> Nothing

833+

834+data Shingle = Shingle

835+ { shingleMaxSize :: Int

836+ , shingleMinSize :: Int

837+ , shingleOutputUnigrams :: Bool

838+ , shingleOutputUnigramsIfNoShingles :: Bool

839+ , shingleTokenSeparator :: Text

840+ , shingleFillerToken :: Text

841+ } deriving (Eq,Show,Generic,Typeable)

626

842

627data TokenizerDefinition

843data TokenizerDefinition

628 = TokenizerDefinitionNgram Ngram

844 = TokenizerDefinitionNgram Ngram

@@ -1110,6 +1326,8 @@ newtype Analyzer =

1110 Analyzer Text deriving (Eq, Read, Show, Generic, ToJSON, FromJSON, Typeable)

1326 Analyzer Text deriving (Eq, Read, Show, Generic, ToJSON, FromJSON, Typeable)

1111newtype Tokenizer =

1327newtype Tokenizer =

1112 Tokenizer Text deriving (Eq, Read, Show, Generic, ToJSON, FromJSON, Typeable)

1328 Tokenizer Text deriving (Eq, Read, Show, Generic, ToJSON, FromJSON, Typeable)

1329+newtype TokenFilter =

1330+ TokenFilter Text deriving (Eq, Read, Show, Generic, ToJSON, FromJSON, Typeable)

1113newtype MaxExpansions =

1331newtype MaxExpansions =

1114 MaxExpansions Int deriving (Eq, Read, Show, Generic, ToJSON, FromJSON, Typeable)

1332 MaxExpansions Int deriving (Eq, Read, Show, Generic, ToJSON, FromJSON, Typeable)

1115

1333

@@ -4751,6 +4969,11 @@ newtype StringlyTypedInt = StringlyTypedInt { unStringlyTypedInt :: Int }

4751instance FromJSON StringlyTypedInt where

4969instance FromJSON StringlyTypedInt where

4752 parseJSON = fmap StringlyTypedInt . parseJSON . unStringlyTypeJSON

4970 parseJSON = fmap StringlyTypedInt . parseJSON . unStringlyTypeJSON

4753

4971

4972+newtype StringlyTypedBool = StringlyTypedBool { unStringlyTypedBool :: Bool }

4973+

4974+instance FromJSON StringlyTypedBool where

4975+ parseJSON = fmap StringlyTypedBool . parseJSON . unStringlyTypeJSON

4976+

4754instance FromJSON NodeFSTotalStats where

4977instance FromJSON NodeFSTotalStats where

4755 parseJSON = withObject "NodeFSTotalStats" parse

4978 parseJSON = withObject "NodeFSTotalStats" parse

4756 where

4979 where

tests/V5/tests.hs+25−1

@@ -911,8 +911,12 @@ instance Arbitrary TokenChar where arbitrary = sopArbitrary; shrink = genericShr

911instance Arbitrary Ngram where arbitrary = sopArbitrary; shrink = genericShrink

912instance Arbitrary TokenizerDefinition where arbitrary = sopArbitrary; shrink = genericShrink

913instance Arbitrary AnalyzerDefinition where arbitrary = sopArbitrary; shrink = genericShrink

914+instance Arbitrary TokenFilterDefinition where arbitrary = sopArbitrary; shrink = genericShrink

915+instance Arbitrary Shingle where arbitrary = sopArbitrary; shrink = genericShrink

916+instance Arbitrary Language where arbitrary = sopArbitrary; shrink = genericShrink

914instance Arbitrary Analysis where arbitrary = sopArbitrary; shrink = genericShrink

917instance Arbitrary Analysis where arbitrary = sopArbitrary; shrink = genericShrink

915instance Arbitrary Tokenizer where arbitrary = sopArbitrary; shrink = genericShrink

918instance Arbitrary Tokenizer where arbitrary = sopArbitrary; shrink = genericShrink

919+instance Arbitrary TokenFilter where arbitrary = sopArbitrary; shrink = genericShrink

916instance Arbitrary UpdatableIndexSetting where arbitrary = sopArbitrary; shrink = genericShrink

920instance Arbitrary UpdatableIndexSetting where arbitrary = sopArbitrary; shrink = genericShrink

917instance Arbitrary Bytes where arbitrary = sopArbitrary; shrink = genericShrink

921instance Arbitrary Bytes where arbitrary = sopArbitrary; shrink = genericShrink

918instance Arbitrary AllocationPolicy where arbitrary = sopArbitrary; shrink = genericShrink

922instance Arbitrary AllocationPolicy where arbitrary = sopArbitrary; shrink = genericShrink

@@ -943,6 +947,7 @@ instance Arbitrary UpdatableIndexSetting' where

943 where

947 where

944 dropDuplicateAttrNames = NE.fromList . L.nubBy sameAttrName . NE.toList

948 dropDuplicateAttrNames = NE.fromList . L.nubBy sameAttrName . NE.toList

945 sameAttrName a b = nodeAttrFilterName a == nodeAttrFilterName b

949 sameAttrName a b = nodeAttrFilterName a == nodeAttrFilterName b

950+ shrink (UpdatableIndexSetting' x) = map UpdatableIndexSetting' (genericShrink x)

946

951

947main :: IO ()

952main :: IO ()

948main = hspec $ do

953main = hspec $ do

@@ -1631,12 +1636,31 @@ main = hspec $ do

1631 it "accepts customer analyzers" $ when' (atleast es50) $ withTestEnv $ do

1636 it "accepts customer analyzers" $ when' (atleast es50) $ withTestEnv $ do

1632 _ <- deleteExampleIndex

1637 _ <- deleteExampleIndex

1633 let analysis = Analysis

1638 let analysis = Analysis

1639+ (M.singleton "ex_analyzer"

1640+ ( AnalyzerDefinition

1634− (M.singleton "ex_analyzer" (AnalyzerDefinition (Just (Tokenizer "ex_tokenizer"))))

1641+ (Just (Tokenizer "ex_tokenizer"))

1642+ (map TokenFilter

1643+ [ "ex_filter_lowercase","ex_filter_uppercase","ex_filter_apostrophe"

1644+ , "ex_filter_reverse","ex_filter_snowball"

1645+ , "ex_filter_shingle"

1646+ ]

1647+ )

1648+ )

1649+ )

1635 (M.singleton "ex_tokenizer"

1650 (M.singleton "ex_tokenizer"

1636 ( TokenizerDefinitionNgram

1651 ( TokenizerDefinitionNgram

1637 ( Ngram 3 4 [TokenLetter,TokenDigit])

1652 ( Ngram 3 4 [TokenLetter,TokenDigit])

1638 )

1653 )

1639 )

1654 )

1655+ (M.fromList

1656+ [ ("ex_filter_lowercase",TokenFilterDefinitionLowercase Greek)

1657+ , ("ex_filter_uppercase",TokenFilterDefinitionUppercase English)

1658+ , ("ex_filter_apostrophe",TokenFilterDefinitionApostrophe)

1659+ , ("ex_filter_reverse",TokenFilterDefinitionReverse)

1660+ , ("ex_filter_snowball",TokenFilterDefinitionSnowball English)

1661+ , ("ex_filter_shingle",TokenFilterDefinitionShingle (Shingle 3 3 True False " " "_"))

1662+ ]

1663+ )

1640 updates = [AnalysisSetting analysis]

1664 updates = [AnalysisSetting analysis]

1641 createResp <- createIndexWith (updates ++ [NumberOfReplicas (ReplicaCount 0)]) 1 testIndex

1665 createResp <- createIndexWith (updates ++ [NumberOfReplicas (ReplicaCount 0)]) 1 testIndex

1642 liftIO $ validateStatus createResp 200

1666 liftIO $ validateStatus createResp 200