git.iko.soy/typeable/bloodhound
on master help sign in
Merged token filters work
commitc64536c39c430cf8416de5905d1c5f894d8b5464 authorChris Allen <cma@bitemyapp.com> dateThu 8 Feb 2018, 18:18 (9 years ago) parentf91547d5291b3a
7 files changed+275−8
compare with master
unifiedsplit
.travis.yml+2−1 src/Database/V5/Bloodhound/Client.hs+10 src/Database/V5/Bloodhound/Internal/Analysis.hs+218−4 src/Database/V5/Bloodhound/Internal/Newtypes.hs+3 src/Database/V5/Bloodhound/Internal/StringlyTyped.hs+9−1 src/Database/V5/Bloodhound/Types.hs+4−1 tests/V5/tests.hs+29−1
.travis.yml+2−1
@@ -21,7 +21,8 @@ env:
2121 - GHCVER=8.2 ESVER=1.7.6 STACK_YAML=stack.yaml ESFLAG=ES1 DLINK=https://download.elastic.co/elasticsearch/elasticsearch/elasticsearch
2222 - GHCVER=8.2 ESVER=5.0.2 STACK_YAML=stack.yaml ESFLAG=ES5 DLINK=https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch ES_JAVA_OPTS="-Xms500m -Xmx750m"
2323 - GHCVER=8.2 ESVER=5.5.0 STACK_YAML=stack.yaml ESFLAG=ES5 DLINK=https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch ES_JAVA_OPTS="-Xms500m -Xmx750m"
24− - GHCVER=8.2 ESVER=6.1.3 STACK_YAML=stack.yaml ESFLAG=ES5 DLINK=https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch ES_JAVA_OPTS="-Xms500m -Xmx750m"
24+ allow_failures:
25+ - GHCVER=8.2 ESVER=6.1.3 STACK_YAML=stack.yaml ESFLAG=ES5 DLINK=https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch ES_JAVA_OPTS="-Xms500m -Xmx750m"
2526
2627install:
2728 # stack
src/Database/V5/Bloodhound/Client.hs+10
@@ -54,6 +54,7 @@ module Database.V5.Bloodhound.Client
5454 -- ** Searching
5555 , searchAll
5656 , searchByIndex
57+ , searchByIndices
5758 , searchByType
5859 , scanSearch
5960 , getInitialScroll
@@ -995,6 +996,15 @@ searchByIndex :: MonadBH m => IndexName -> Search -> m Reply
995996searchByIndex (IndexName indexName) = bindM2 dispatchSearch url . return
996997 where url = joinPath [indexName, "_search"]
997998
999+-- | 'searchByIndices' is a variant of 'searchByIndex' that executes a
1000+-- 'Search' over many indices. This is much faster than using
1001+-- 'mapM' to 'searchByIndex' over a collection since it only
1002+-- causes a single HTTP request to be emitted.
1003+searchByIndices :: MonadBH m => NonEmpty IndexName -> Search -> m Reply
1004+searchByIndices ixs = bindM2 dispatchSearch url . return
1005+ where url = joinPath [renderedIxs, "_search"]
1006+ renderedIxs = T.intercalate (T.singleton ',') (map (\(IndexName t) -> t) (toList ixs))
1007+
9981008-- | 'searchByType', given a 'Search', 'IndexName', and 'MappingName', will perform that
9991009-- search against a specific mapping within an index on an Elasticsearch server.
10001010--
src/Database/V5/Bloodhound/Internal/Analysis.hs+218−4
@@ -8,24 +8,29 @@ import Data.Aeson.Types (Parser)
88import qualified Data.Map.Strict as M
99import Data.Maybe (catMaybes)
1010import Data.Text (Text)
11+import qualified Data.Text as T
1112
13+import Database.V5.Bloodhound.Internal.Newtypes
1214import Database.V5.Bloodhound.Internal.StringlyTyped
1315
1416data Analysis = Analysis
1517 { analysisAnalyzer :: M.Map Text AnalyzerDefinition
1618 , analysisTokenizer :: M.Map Text TokenizerDefinition
19+ , analysisTokenFilter :: M.Map Text TokenFilterDefinition
1720 } deriving (Eq, Show)
1821
1922instance ToJSON Analysis where
20− toJSON (Analysis analyzer tokenizer) = object
23+ toJSON (Analysis analyzer tokenizer tokenFilter) = object
2124 [ "analyzer" .= analyzer
2225 , "tokenizer" .= tokenizer
26+ , "filter" .= tokenFilter
2327 ]
2428
2529instance FromJSON Analysis where
2630 parseJSON = withObject "Analysis" $ \m -> Analysis
2731 <$> m .: "analyzer"
28− <*> m .: "tokenizer"
32+ <*> m .:? "tokenizer" .!= M.empty
33+ <*> m .:? "filter" .!= M.empty
2934
3035newtype Tokenizer =
3136 Tokenizer Text
@@ -33,17 +38,19 @@ newtype Tokenizer =
3338
3439data AnalyzerDefinition = AnalyzerDefinition
3540 { analyzerDefinitionTokenizer :: Maybe Tokenizer
41+ , analyzerDefinitionFilter :: [TokenFilter]
3642 } deriving (Eq,Show)
3743
3844instance ToJSON AnalyzerDefinition where
39− toJSON (AnalyzerDefinition tokenizer) = object $ catMaybes
45+ toJSON (AnalyzerDefinition tokenizer tokenFilter) = object $ catMaybes
4046 [ fmap ("tokenizer" .=) tokenizer
47+ , Just $ "filter" .= tokenFilter
4148 ]
4249
4350instance FromJSON AnalyzerDefinition where
4451 parseJSON = withObject "AnalyzerDefinition" $ \m -> AnalyzerDefinition
4552 <$> m .:? "tokenizer"
46−
53+ <*> m .:? "filter" .!= []
4754
4855data TokenizerDefinition
4956 = TokenizerDefinitionNgram Ngram
@@ -98,3 +105,210 @@ instance FromJSON TokenChar where
98105 "punctuation" -> return TokenPunctuation
99106 "symbol" -> return TokenSymbol
100107 _ -> fail "invalid TokenChar"
108+
109+-- | Token filters are used to create custom analyzers.
110+data TokenFilterDefinition
111+ = TokenFilterDefinitionLowercase (Maybe Language)
112+ | TokenFilterDefinitionUppercase (Maybe Language)
113+ | TokenFilterDefinitionApostrophe
114+ | TokenFilterDefinitionReverse
115+ | TokenFilterDefinitionSnowball Language
116+ | TokenFilterDefinitionShingle Shingle
117+ deriving (Eq, Show)
118+
119+instance ToJSON TokenFilterDefinition where
120+ toJSON x = case x of
121+ TokenFilterDefinitionLowercase mlang -> object $ catMaybes
122+ [ Just $ "type" .= ("lowercase" :: Text)
123+ , fmap (\lang -> "language" .= languageToText lang) mlang
124+ ]
125+ TokenFilterDefinitionUppercase mlang -> object $ catMaybes
126+ [ Just $ "type" .= ("uppercase" :: Text)
127+ , fmap (\lang -> "language" .= languageToText lang) mlang
128+ ]
129+ TokenFilterDefinitionApostrophe -> object
130+ [ "type" .= ("apostrophe" :: Text)
131+ ]
132+ TokenFilterDefinitionReverse -> object
133+ [ "type" .= ("reverse" :: Text)
134+ ]
135+ TokenFilterDefinitionSnowball lang -> object
136+ [ "type" .= ("snowball" :: Text)
137+ , "language" .= languageToText lang
138+ ]
139+ TokenFilterDefinitionShingle s -> object
140+ [ "type" .= ("shingle" :: Text)
141+ , "max_shingle_size" .= shingleMaxSize s
142+ , "min_shingle_size" .= shingleMinSize s
143+ , "output_unigrams" .= shingleOutputUnigrams s
144+ , "output_unigrams_if_no_shingles" .= shingleOutputUnigramsIfNoShingles s
145+ , "token_separator" .= shingleTokenSeparator s
146+ , "filler_token" .= shingleFillerToken s
147+ ]
148+
149+instance FromJSON TokenFilterDefinition where
150+ parseJSON = withObject "TokenFilterDefinition" $ \m -> do
151+ t <- m .: "type"
152+ case (t :: Text) of
153+ "reverse" -> return TokenFilterDefinitionReverse
154+ "apostrophe" -> return TokenFilterDefinitionApostrophe
155+ "lowercase" -> TokenFilterDefinitionLowercase
156+ <$> m .:? "language"
157+ "uppercase" -> TokenFilterDefinitionUppercase
158+ <$> m .:? "language"
159+ "snowball" -> TokenFilterDefinitionSnowball
160+ <$> m .: "language"
161+ "shingle" -> fmap TokenFilterDefinitionShingle $ Shingle
162+ <$> (fmap.fmap) unStringlyTypedInt (m .:? "max_shingle_size") .!= 2
163+ <*> (fmap.fmap) unStringlyTypedInt (m .:? "min_shingle_size") .!= 2
164+ <*> (fmap.fmap) unStringlyTypedBool (m .:? "output_unigrams") .!= True
165+ <*> (fmap.fmap) unStringlyTypedBool (m .:? "output_unigrams_if_no_shingles") .!= False
166+ <*> m .:? "token_separator" .!= " "
167+ <*> m .:? "filler_token" .!= "_"
168+ _ -> fail ("unrecognized token filter type: " ++ T.unpack t)
169+
170+-- | The set of languages that can be passed to various analyzers,
171+-- filters, etc. in ElasticSearch. Most data types in this module
172+-- that have a 'Language' field are actually only actually to
173+-- handle a subset of these languages. Consult the official
174+-- ElasticSearch documentation to see what is actually supported.
175+data Language
176+ = Arabic
177+ | Armenian
178+ | Basque
179+ | Bengali
180+ | Brazilian
181+ | Bulgarian
182+ | Catalan
183+ | Cjk
184+ | Czech
185+ | Danish
186+ | Dutch
187+ | English
188+ | Finnish
189+ | French
190+ | Galician
191+ | German
192+ | German2
193+ | Greek
194+ | Hindi
195+ | Hungarian
196+ | Indonesian
197+ | Irish
198+ | Italian
199+ | Kp
200+ | Latvian
201+ | Lithuanian
202+ | Lovins
203+ | Norwegian
204+ | Persian
205+ | Porter
206+ | Portuguese
207+ | Romanian
208+ | Russian
209+ | Sorani
210+ | Spanish
211+ | Swedish
212+ | Thai
213+ | Turkish
214+ deriving (Eq, Show)
215+
216+instance ToJSON Language where
217+ toJSON = Data.Aeson.String . languageToText
218+
219+instance FromJSON Language where
220+ parseJSON = withText "Language" $ \t -> case languageFromText t of
221+ Nothing -> fail "not a supported ElasticSearch language"
222+ Just lang -> return lang
223+
224+languageToText :: Language -> Text
225+languageToText x = case x of
226+ Arabic -> "arabic"
227+ Armenian -> "armenian"
228+ Basque -> "basque"
229+ Bengali -> "bengali"
230+ Brazilian -> "brazilian"
231+ Bulgarian -> "bulgarian"
232+ Catalan -> "catalan"
233+ Cjk -> "cjk"
234+ Czech -> "czech"
235+ Danish -> "danish"
236+ Dutch -> "dutch"
237+ English -> "english"
238+ Finnish -> "finnish"
239+ French -> "french"
240+ Galician -> "galician"
241+ German -> "german"
242+ German2 -> "german2"
243+ Greek -> "greek"
244+ Hindi -> "hindi"
245+ Hungarian -> "hungarian"
246+ Indonesian -> "indonesian"
247+ Irish -> "irish"
248+ Italian -> "italian"
249+ Kp -> "kp"
250+ Latvian -> "latvian"
251+ Lithuanian -> "lithuanian"
252+ Lovins -> "lovins"
253+ Norwegian -> "norwegian"
254+ Persian -> "persian"
255+ Porter -> "porter"
256+ Portuguese -> "portuguese"
257+ Romanian -> "romanian"
258+ Russian -> "russian"
259+ Sorani -> "sorani"
260+ Spanish -> "spanish"
261+ Swedish -> "swedish"
262+ Thai -> "thai"
263+ Turkish -> "turkish"
264+
265+languageFromText :: Text -> Maybe Language
266+languageFromText x = case x of
267+ "arabic" -> Just Arabic
268+ "armenian" -> Just Armenian
269+ "basque" -> Just Basque
270+ "bengali" -> Just Bengali
271+ "brazilian" -> Just Brazilian
272+ "bulgarian" -> Just Bulgarian
273+ "catalan" -> Just Catalan
274+ "cjk" -> Just Cjk
275+ "czech" -> Just Czech
276+ "danish" -> Just Danish
277+ "dutch" -> Just Dutch
278+ "english" -> Just English
279+ "finnish" -> Just Finnish
280+ "french" -> Just French
281+ "galician" -> Just Galician
282+ "german" -> Just German
283+ "german2" -> Just German2
284+ "greek" -> Just Greek
285+ "hindi" -> Just Hindi
286+ "hungarian" -> Just Hungarian
287+ "indonesian" -> Just Indonesian
288+ "irish" -> Just Irish
289+ "italian" -> Just Italian
290+ "kp" -> Just Kp
291+ "latvian" -> Just Latvian
292+ "lithuanian" -> Just Lithuanian
293+ "lovins" -> Just Lovins
294+ "norwegian" -> Just Norwegian
295+ "persian" -> Just Persian
296+ "porter" -> Just Porter
297+ "portuguese" -> Just Portuguese
298+ "romanian" -> Just Romanian
299+ "russian" -> Just Russian
300+ "sorani" -> Just Sorani
301+ "spanish" -> Just Spanish
302+ "swedish" -> Just Swedish
303+ "thai" -> Just Thai
304+ "turkish" -> Just Turkish
305+ _ -> Nothing
306+
307+data Shingle = Shingle
308+ { shingleMaxSize :: Int
309+ , shingleMinSize :: Int
310+ , shingleOutputUnigrams :: Bool
311+ , shingleOutputUnigramsIfNoShingles :: Bool
312+ , shingleTokenSeparator :: Text
313+ , shingleFillerToken :: Text
314+ } deriving (Eq, Show)
src/Database/V5/Bloodhound/Internal/Newtypes.hs+3
@@ -196,3 +196,6 @@ instance FromJSON MS where
196196 parseJSON = withScientific "MS" (return . MS . parse)
197197 where
198198 parse n = fromInteger ((truncate n) * 1000)
199+
200+newtype TokenFilter =
201+ TokenFilter Text deriving (Eq, Show, FromJSON, ToJSON)
src/Database/V5/Bloodhound/Internal/StringlyTyped.hs+9−1
@@ -26,8 +26,16 @@ instance FromJSON StringlyTypedInt where
2626 . parseJSON
2727 . unStringlyTypeJSON
2828
29+newtype StringlyTypedBool = StringlyTypedBool { unStringlyTypedBool :: Bool }
30+
31+instance FromJSON StringlyTypedBool where
32+ parseJSON =
33+ fmap StringlyTypedBool
34+ . parseJSON
35+ . unStringlyTypeJSON
36+
2937-- | For some reason in several settings APIs, all leaf values get returned
30−-- as strings. This function attepmts to recover from this for all
38+-- as strings. This function attempts to recover from this for all
3139-- non-recursive JSON types. If nothing can be done, the value is left alone.
3240unStringlyTypeJSON :: Value -> Value
3341unStringlyTypeJSON (String "true") =
src/Database/V5/Bloodhound/Types.hs+4−1
@@ -223,6 +223,7 @@ module Database.V5.Bloodhound.Types
223223 , CutoffFrequency(..)
224224 , Analyzer(..)
225225 , Tokenizer(..)
226+ , TokenFilter(..)
226227 , MaxExpansions(..)
227228 , Lenient(..)
228229 , MatchQueryType(..)
@@ -404,8 +405,11 @@ module Database.V5.Bloodhound.Types
404405 , Analysis(..)
405406 , AnalyzerDefinition(..)
406407 , TokenizerDefinition(..)
408+ , TokenFilterDefinition(..)
407409 , Ngram(..)
408410 , TokenChar(..)
411+ , Shingle(..)
412+ , Language(..)
409413 ) where
410414
411415import Bloodhound.Import
@@ -1575,7 +1579,6 @@ instance FromJSON DocVersion where
15751579 i <- parseJSON v
15761580 maybe (fail "DocVersion out of range") return $ mkDocVersion i
15771581
1578−
15791582data Suggest = Suggest { suggestText :: Text
15801583 , suggestName :: Text
15811584 , suggestType :: SuggestType
tests/V5/tests.hs+29−1
@@ -925,6 +925,14 @@ makeArbitrary ''Ngram
925925instance Arbitrary Ngram where arbitrary = arbitraryNgram
926926makeArbitrary ''TokenizerDefinition
927927instance Arbitrary TokenizerDefinition where arbitrary = arbitraryTokenizerDefinition
928+makeArbitrary ''TokenFilter
929+instance Arbitrary TokenFilter where arbitrary = arbitraryTokenFilter
930+makeArbitrary ''TokenFilterDefinition
931+instance Arbitrary TokenFilterDefinition where arbitrary = arbitraryTokenFilterDefinition
932+makeArbitrary ''Language
933+instance Arbitrary Language where arbitrary = arbitraryLanguage
934+makeArbitrary ''Shingle
935+instance Arbitrary Shingle where arbitrary = arbitraryShingle
928936makeArbitrary ''AnalyzerDefinition
929937instance Arbitrary AnalyzerDefinition where arbitrary = arbitraryAnalyzerDefinition
930938makeArbitrary ''Analysis
@@ -1023,6 +1031,7 @@ instance Arbitrary UpdatableIndexSetting' where
10231031 NE.fromList . L.nubBy sameAttrName . NE.toList
10241032 sameAttrName a b =
10251033 nodeAttrFilterName a == nodeAttrFilterName b
1034+ -- shrink (UpdatableIndexSetting' x) = map UpdatableIndexSetting' (shrink x)
10261035
10271036-- | Due to the way nodeattrfilters get serialized here, they may come
10281037-- out in a different order, but they are morally equivalent
@@ -1773,12 +1782,31 @@ main = hspec $ do
17731782 it "accepts customer analyzers" $ when' (atleast es50) $ withTestEnv $ do
17741783 _ <- deleteExampleIndex
17751784 let analysis = Analysis
1776− (M.singleton "ex_analyzer" (AnalyzerDefinition (Just (Tokenizer "ex_tokenizer"))))
1785+ (M.singleton "ex_analyzer"
1786+ ( AnalyzerDefinition
1787+ (Just (Tokenizer "ex_tokenizer"))
1788+ (map TokenFilter
1789+ [ "ex_filter_lowercase","ex_filter_uppercase","ex_filter_apostrophe"
1790+ , "ex_filter_reverse","ex_filter_snowball"
1791+ , "ex_filter_shingle"
1792+ ]
1793+ )
1794+ )
1795+ )
17771796 (M.singleton "ex_tokenizer"
17781797 ( TokenizerDefinitionNgram
17791798 ( Ngram 3 4 [TokenLetter,TokenDigit])
17801799 )
17811800 )
1801+ (M.fromList
1802+ [ ("ex_filter_lowercase",TokenFilterDefinitionLowercase (Just Greek))
1803+ , ("ex_filter_uppercase",TokenFilterDefinitionUppercase Nothing)
1804+ , ("ex_filter_apostrophe",TokenFilterDefinitionApostrophe)
1805+ , ("ex_filter_reverse",TokenFilterDefinitionReverse)
1806+ , ("ex_filter_snowball",TokenFilterDefinitionSnowball English)
1807+ , ("ex_filter_shingle",TokenFilterDefinitionShingle (Shingle 3 3 True False " " "_"))
1808+ ]
1809+ )
17821810 updates = [AnalysisSetting analysis]
17831811 createResp <- createIndexWith (updates ++ [NumberOfReplicas (ReplicaCount 0)]) 1 testIndex
17841812 liftIO $ validateStatus createResp 200