b161cad · typeable/bloodhound

git.iko.soy/typeable/bloodhound

on master help sign in

Merge pull request #219 from andrewthad/basic_token_filter

b161cadChris Allen9 years ago

src/Database/V5/Bloodhound/Types.hs+226−3

compare with master

unifiedsplit

@@ -204,6 +204,7 @@ module Database.V5.Bloodhound.Types

204 , CutoffFrequency(..)

205 , Analyzer(..)

206 , Tokenizer(..)

207+ , TokenFilter(..)

207 , MaxExpansions(..)

208 , MaxExpansions(..)

208 , Lenient(..)

209 , Lenient(..)

209 , MatchQueryType(..)

210 , MatchQueryType(..)

@@ -385,8 +386,11 @@ module Database.V5.Bloodhound.Types

385 , Analysis(..)

386 , Analysis(..)

386 , AnalyzerDefinition(..)

387 , AnalyzerDefinition(..)

387 , TokenizerDefinition(..)

388 , TokenizerDefinition(..)

389+ , TokenFilterDefinition(..)

388 , Ngram(..)

390 , Ngram(..)

389 , TokenChar(..)

391 , TokenChar(..)

392+ , Shingle(..)

393+ , Language(..)

390 ) where

394 ) where

391

395

392import Control.Applicative as A

396import Control.Applicative as A

@@ -602,32 +606,244 @@ data UpdatableIndexSetting = NumberOfReplicas ReplicaCount

602data Analysis = Analysis

606data Analysis = Analysis

603 { analysisAnalyzer :: M.Map Text AnalyzerDefinition

607 { analysisAnalyzer :: M.Map Text AnalyzerDefinition

604 , analysisTokenizer :: M.Map Text TokenizerDefinition

608 , analysisTokenizer :: M.Map Text TokenizerDefinition

609+ , analysisTokenFilter :: M.Map Text TokenFilterDefinition

605 } deriving (Eq,Show,Generic,Typeable)

610 } deriving (Eq,Show,Generic,Typeable)

606

611

607instance ToJSON Analysis where

612instance ToJSON Analysis where

608− toJSON (Analysis analyzer tokenizer) = object

613+ toJSON (Analysis analyzer tokenizer tokenFilter) = object

609 [ "analyzer" .= analyzer

614 [ "analyzer" .= analyzer

610 , "tokenizer" .= tokenizer

615 , "tokenizer" .= tokenizer

616+ , "filter" .= tokenFilter

611 ]

617 ]

612

618

613instance FromJSON Analysis where

619instance FromJSON Analysis where

614 parseJSON = withObject "Analysis" $ \m -> Analysis

620 parseJSON = withObject "Analysis" $ \m -> Analysis

615 <$> m .: "analyzer"

621 <$> m .: "analyzer"

616− <*> m .: "tokenizer"

622+ <*> m .:? "tokenizer" .!= M.empty

623+ <*> m .:? "filter" .!= M.empty

617

624

618data AnalyzerDefinition = AnalyzerDefinition

625data AnalyzerDefinition = AnalyzerDefinition

619 { analyzerDefinitionTokenizer :: Maybe Tokenizer

626 { analyzerDefinitionTokenizer :: Maybe Tokenizer

627+ , analyzerDefinitionFilter :: [TokenFilter]

620 } deriving (Eq,Show,Generic,Typeable)

628 } deriving (Eq,Show,Generic,Typeable)

621

629

622instance ToJSON AnalyzerDefinition where

630instance ToJSON AnalyzerDefinition where

623− toJSON (AnalyzerDefinition tokenizer) = object $ catMaybes

631+ toJSON (AnalyzerDefinition tokenizer tokenFilter) = object $ catMaybes

624 [ fmap ("tokenizer" .=) tokenizer

632 [ fmap ("tokenizer" .=) tokenizer

633+ , Just $ "filter" .= tokenFilter

625 ]

634 ]

626

635

627instance FromJSON AnalyzerDefinition where

636instance FromJSON AnalyzerDefinition where

628 parseJSON = withObject "AnalyzerDefinition" $ \m -> AnalyzerDefinition

637 parseJSON = withObject "AnalyzerDefinition" $ \m -> AnalyzerDefinition

629 <$> m .:? "tokenizer"

638 <$> m .:? "tokenizer"

639+ <*> m .:? "filter" .!= []

630

640

641+-- | Token filters are used to create custom analyzers.

642+data TokenFilterDefinition

643+ = TokenFilterDefinitionLowercase (Maybe Language)

644+ | TokenFilterDefinitionUppercase (Maybe Language)

645+ | TokenFilterDefinitionApostrophe

646+ | TokenFilterDefinitionReverse

647+ | TokenFilterDefinitionSnowball Language

648+ | TokenFilterDefinitionShingle Shingle

649+ deriving (Eq,Show,Generic)

650+

651+instance ToJSON TokenFilterDefinition where

652+ toJSON x = case x of

653+ TokenFilterDefinitionLowercase mlang -> object $ catMaybes

654+ [ Just $ "type" .= ("lowercase" :: Text)

655+ , fmap (\lang -> "language" .= languageToText lang) mlang

656+ ]

657+ TokenFilterDefinitionUppercase mlang -> object $ catMaybes

658+ [ Just $ "type" .= ("uppercase" :: Text)

659+ , fmap (\lang -> "language" .= languageToText lang) mlang

660+ ]

661+ TokenFilterDefinitionApostrophe -> object

662+ [ "type" .= ("apostrophe" :: Text)

663+ ]

664+ TokenFilterDefinitionReverse -> object

665+ [ "type" .= ("reverse" :: Text)

666+ ]

667+ TokenFilterDefinitionSnowball lang -> object

668+ [ "type" .= ("snowball" :: Text)

669+ , "language" .= languageToText lang

670+ ]

671+ TokenFilterDefinitionShingle s -> object

672+ [ "type" .= ("shingle" :: Text)

673+ , "max_shingle_size" .= shingleMaxSize s

674+ , "min_shingle_size" .= shingleMinSize s

675+ , "output_unigrams" .= shingleOutputUnigrams s

676+ , "output_unigrams_if_no_shingles" .= shingleOutputUnigramsIfNoShingles s

677+ , "token_separator" .= shingleTokenSeparator s

678+ , "filler_token" .= shingleFillerToken s

679+ ]

680+

681+instance FromJSON TokenFilterDefinition where

682+ parseJSON = withObject "TokenFilterDefinition" $ \m -> do

683+ t <- m .: "type"

684+ case (t :: Text) of

685+ "reverse" -> return TokenFilterDefinitionReverse

686+ "apostrophe" -> return TokenFilterDefinitionApostrophe

687+ "lowercase" -> TokenFilterDefinitionLowercase

688+ <$> m .:? "language"

689+ "uppercase" -> TokenFilterDefinitionUppercase

690+ <$> m .:? "language"

691+ "snowball" -> TokenFilterDefinitionSnowball

692+ <$> m .: "language"

693+ "shingle" -> fmap TokenFilterDefinitionShingle $ Shingle

694+ <$> (fmap.fmap) unStringlyTypedInt (m .:? "max_shingle_size") .!= 2

695+ <*> (fmap.fmap) unStringlyTypedInt (m .:? "min_shingle_size") .!= 2

696+ <*> (fmap.fmap) unStringlyTypedBool (m .:? "output_unigrams") .!= True

697+ <*> (fmap.fmap) unStringlyTypedBool (m .:? "output_unigrams_if_no_shingles") .!= False

698+ <*> m .:? "token_separator" .!= " "

699+ <*> m .:? "filler_token" .!= "_"

700+ _ -> fail ("unrecognized token filter type: " ++ T.unpack t)

701+

702+-- | The set of languages that can be passed to various analyzers,

703+-- filters, etc. in ElasticSearch. Most data types in this module

704+-- that have a 'Language' field are actually only actually to

705+-- handle a subset of these languages. Consult the official

706+-- ElasticSearch documentation to see what is actually supported.

707+data Language

708+ = Arabic

709+ | Armenian

710+ | Basque

711+ | Bengali

712+ | Brazilian

713+ | Bulgarian

714+ | Catalan

715+ | Cjk

716+ | Czech

717+ | Danish

718+ | Dutch

719+ | English

720+ | Finnish

721+ | French

722+ | Galician

723+ | German

724+ | German2

725+ | Greek

726+ | Hindi

727+ | Hungarian

728+ | Indonesian

729+ | Irish

730+ | Italian

731+ | Kp

732+ | Latvian

733+ | Lithuanian

734+ | Lovins

735+ | Norwegian

736+ | Persian

737+ | Porter

738+ | Portuguese

739+ | Romanian

740+ | Russian

741+ | Sorani

742+ | Spanish

743+ | Swedish

744+ | Thai

745+ | Turkish

746+ deriving (Show,Eq,Generic)

747+

748+instance ToJSON Language where

749+ toJSON = Data.Aeson.String . languageToText

750+

751+instance FromJSON Language where

752+ parseJSON = withText "Language" $ \t -> case languageFromText t of

753+ Nothing -> fail "not a supported ElasticSearch language"

754+ Just lang -> return lang

755+

756+languageToText :: Language -> Text

757+languageToText x = case x of

758+ Arabic -> "arabic"

759+ Armenian -> "armenian"

760+ Basque -> "basque"

761+ Bengali -> "bengali"

762+ Brazilian -> "brazilian"

763+ Bulgarian -> "bulgarian"

764+ Catalan -> "catalan"

765+ Cjk -> "cjk"

766+ Czech -> "czech"

767+ Danish -> "danish"

768+ Dutch -> "dutch"

769+ English -> "english"

770+ Finnish -> "finnish"

771+ French -> "french"

772+ Galician -> "galician"

773+ German -> "german"

774+ German2 -> "german2"

775+ Greek -> "greek"

776+ Hindi -> "hindi"

777+ Hungarian -> "hungarian"

778+ Indonesian -> "indonesian"

779+ Irish -> "irish"

780+ Italian -> "italian"

781+ Kp -> "kp"

782+ Latvian -> "latvian"

783+ Lithuanian -> "lithuanian"

784+ Lovins -> "lovins"

785+ Norwegian -> "norwegian"

786+ Persian -> "persian"

787+ Porter -> "porter"

788+ Portuguese -> "portuguese"

789+ Romanian -> "romanian"

790+ Russian -> "russian"

791+ Sorani -> "sorani"

792+ Spanish -> "spanish"

793+ Swedish -> "swedish"

794+ Thai -> "thai"

795+ Turkish -> "turkish"

796+

797+languageFromText :: Text -> Maybe Language

798+languageFromText x = case x of

799+ "arabic" -> Just Arabic

800+ "armenian" -> Just Armenian

801+ "basque" -> Just Basque

802+ "bengali" -> Just Bengali

803+ "brazilian" -> Just Brazilian

804+ "bulgarian" -> Just Bulgarian

805+ "catalan" -> Just Catalan

806+ "cjk" -> Just Cjk

807+ "czech" -> Just Czech

808+ "danish" -> Just Danish

809+ "dutch" -> Just Dutch

810+ "english" -> Just English

811+ "finnish" -> Just Finnish

812+ "french" -> Just French

813+ "galician" -> Just Galician

814+ "german" -> Just German

815+ "german2" -> Just German2

816+ "greek" -> Just Greek

817+ "hindi" -> Just Hindi

818+ "hungarian" -> Just Hungarian

819+ "indonesian" -> Just Indonesian

820+ "irish" -> Just Irish

821+ "italian" -> Just Italian

822+ "kp" -> Just Kp

823+ "latvian" -> Just Latvian

824+ "lithuanian" -> Just Lithuanian

825+ "lovins" -> Just Lovins

826+ "norwegian" -> Just Norwegian

827+ "persian" -> Just Persian

828+ "porter" -> Just Porter

829+ "portuguese" -> Just Portuguese

830+ "romanian" -> Just Romanian

831+ "russian" -> Just Russian

832+ "sorani" -> Just Sorani

833+ "spanish" -> Just Spanish

834+ "swedish" -> Just Swedish

835+ "thai" -> Just Thai

836+ "turkish" -> Just Turkish

837+ _ -> Nothing

838+

839+data Shingle = Shingle

840+ { shingleMaxSize :: Int

841+ , shingleMinSize :: Int

842+ , shingleOutputUnigrams :: Bool

843+ , shingleOutputUnigramsIfNoShingles :: Bool

844+ , shingleTokenSeparator :: Text

845+ , shingleFillerToken :: Text

846+ } deriving (Eq,Show,Generic,Typeable)

631

847

632data TokenizerDefinition

848data TokenizerDefinition

633 = TokenizerDefinitionNgram Ngram

849 = TokenizerDefinitionNgram Ngram

@@ -1135,6 +1351,8 @@ newtype Analyzer =

1135 Analyzer Text deriving (Eq, Read, Show, Generic, ToJSON, FromJSON, Typeable)

1351 Analyzer Text deriving (Eq, Read, Show, Generic, ToJSON, FromJSON, Typeable)

1136newtype Tokenizer =

1352newtype Tokenizer =

1137 Tokenizer Text deriving (Eq, Read, Show, Generic, ToJSON, FromJSON, Typeable)

1353 Tokenizer Text deriving (Eq, Read, Show, Generic, ToJSON, FromJSON, Typeable)

1354+newtype TokenFilter =

1355+ TokenFilter Text deriving (Eq, Read, Show, Generic, ToJSON, FromJSON, Typeable)

1138newtype MaxExpansions =

1356newtype MaxExpansions =

1139 MaxExpansions Int deriving (Eq, Read, Show, Generic, ToJSON, FromJSON, Typeable)

1357 MaxExpansions Int deriving (Eq, Read, Show, Generic, ToJSON, FromJSON, Typeable)

1140

1358

@@ -4779,6 +4997,11 @@ newtype StringlyTypedInt = StringlyTypedInt { unStringlyTypedInt :: Int }

4779instance FromJSON StringlyTypedInt where

4997instance FromJSON StringlyTypedInt where

4780 parseJSON = fmap StringlyTypedInt . parseJSON . unStringlyTypeJSON

4998 parseJSON = fmap StringlyTypedInt . parseJSON . unStringlyTypeJSON

4781

4999

5000+newtype StringlyTypedBool = StringlyTypedBool { unStringlyTypedBool :: Bool }

5001+

5002+instance FromJSON StringlyTypedBool where

5003+ parseJSON = fmap StringlyTypedBool . parseJSON . unStringlyTypeJSON

5004+

4782instance FromJSON NodeFSTotalStats where

5005instance FromJSON NodeFSTotalStats where

4783 parseJSON = withObject "NodeFSTotalStats" parse

5006 parseJSON = withObject "NodeFSTotalStats" parse

4784 where

5007 where

tests/V5/tests.hs+25−1

@@ -925,8 +925,12 @@ instance Arbitrary TokenChar where arbitrary = sopArbitrary; shrink = genericShr

925instance Arbitrary Ngram where arbitrary = sopArbitrary; shrink = genericShrink

926instance Arbitrary TokenizerDefinition where arbitrary = sopArbitrary; shrink = genericShrink

927instance Arbitrary AnalyzerDefinition where arbitrary = sopArbitrary; shrink = genericShrink

928+instance Arbitrary TokenFilterDefinition where arbitrary = sopArbitrary; shrink = genericShrink

929+instance Arbitrary Shingle where arbitrary = sopArbitrary; shrink = genericShrink

930+instance Arbitrary Language where arbitrary = sopArbitrary; shrink = genericShrink

928instance Arbitrary Analysis where arbitrary = sopArbitrary; shrink = genericShrink

931instance Arbitrary Analysis where arbitrary = sopArbitrary; shrink = genericShrink

929instance Arbitrary Tokenizer where arbitrary = sopArbitrary; shrink = genericShrink

932instance Arbitrary Tokenizer where arbitrary = sopArbitrary; shrink = genericShrink

933+instance Arbitrary TokenFilter where arbitrary = sopArbitrary; shrink = genericShrink

930instance Arbitrary UpdatableIndexSetting where arbitrary = sopArbitrary; shrink = genericShrink

934instance Arbitrary UpdatableIndexSetting where arbitrary = sopArbitrary; shrink = genericShrink

931instance Arbitrary Compression where arbitrary = sopArbitrary; shrink = genericShrink

935instance Arbitrary Compression where arbitrary = sopArbitrary; shrink = genericShrink

932instance Arbitrary Bytes where arbitrary = sopArbitrary; shrink = genericShrink

936instance Arbitrary Bytes where arbitrary = sopArbitrary; shrink = genericShrink

@@ -960,6 +964,7 @@ instance Arbitrary UpdatableIndexSetting' where

960 where

964 where

961 dropDuplicateAttrNames = NE.fromList . L.nubBy sameAttrName . NE.toList

965 dropDuplicateAttrNames = NE.fromList . L.nubBy sameAttrName . NE.toList

962 sameAttrName a b = nodeAttrFilterName a == nodeAttrFilterName b

966 sameAttrName a b = nodeAttrFilterName a == nodeAttrFilterName b

967+ shrink (UpdatableIndexSetting' x) = map UpdatableIndexSetting' (genericShrink x)

963

968

964main :: IO ()

969main :: IO ()

965main = hspec $ do

970main = hspec $ do

@@ -1648,12 +1653,31 @@ main = hspec $ do

1648 it "accepts customer analyzers" $ when' (atleast es50) $ withTestEnv $ do

1653 it "accepts customer analyzers" $ when' (atleast es50) $ withTestEnv $ do

1649 _ <- deleteExampleIndex

1654 _ <- deleteExampleIndex

1650 let analysis = Analysis

1655 let analysis = Analysis

1656+ (M.singleton "ex_analyzer"

1657+ ( AnalyzerDefinition

1651− (M.singleton "ex_analyzer" (AnalyzerDefinition (Just (Tokenizer "ex_tokenizer"))))

1658+ (Just (Tokenizer "ex_tokenizer"))

1659+ (map TokenFilter

1660+ [ "ex_filter_lowercase","ex_filter_uppercase","ex_filter_apostrophe"

1661+ , "ex_filter_reverse","ex_filter_snowball"

1662+ , "ex_filter_shingle"

1663+ ]

1664+ )

1665+ )

1666+ )

1652 (M.singleton "ex_tokenizer"

1667 (M.singleton "ex_tokenizer"

1653 ( TokenizerDefinitionNgram

1668 ( TokenizerDefinitionNgram

1654 ( Ngram 3 4 [TokenLetter,TokenDigit])

1669 ( Ngram 3 4 [TokenLetter,TokenDigit])

1655 )

1670 )

1656 )

1671 )

1672+ (M.fromList

1673+ [ ("ex_filter_lowercase",TokenFilterDefinitionLowercase (Just Greek))

1674+ , ("ex_filter_uppercase",TokenFilterDefinitionUppercase Nothing)

1675+ , ("ex_filter_apostrophe",TokenFilterDefinitionApostrophe)

1676+ , ("ex_filter_reverse",TokenFilterDefinitionReverse)

1677+ , ("ex_filter_snowball",TokenFilterDefinitionSnowball English)

1678+ , ("ex_filter_shingle",TokenFilterDefinitionShingle (Shingle 3 3 True False " " "_"))

1679+ ]

1680+ )

1657 updates = [AnalysisSetting analysis]

1681 updates = [AnalysisSetting analysis]

1658 createResp <- createIndexWith (updates ++ [NumberOfReplicas (ReplicaCount 0)]) 1 testIndex

1682 createResp <- createIndexWith (updates ++ [NumberOfReplicas (ReplicaCount 0)]) 1 testIndex

1659 liftIO $ validateStatus createResp 200

1683 liftIO $ validateStatus createResp 200