c64536c · typeable/bloodhound

git.iko.soy/typeable/bloodhound

Merged token filters work

c64536cChris Allen9 years ago

.travis.yml+2−1

@@ -21,7 +21,8 @@ env:

21 - GHCVER=8.2 ESVER=1.7.6 STACK_YAML=stack.yaml ESFLAG=ES1 DLINK=https://download.elastic.co/elasticsearch/elasticsearch/elasticsearch

22 - GHCVER=8.2 ESVER=5.0.2 STACK_YAML=stack.yaml ESFLAG=ES5 DLINK=https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch ES_JAVA_OPTS="-Xms500m -Xmx750m"

23 - GHCVER=8.2 ESVER=5.5.0 STACK_YAML=stack.yaml ESFLAG=ES5 DLINK=https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch ES_JAVA_OPTS="-Xms500m -Xmx750m"

24+ allow_failures:

24− - GHCVER=8.2 ESVER=6.1.3 STACK_YAML=stack.yaml ESFLAG=ES5 DLINK=https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch ES_JAVA_OPTS="-Xms500m -Xmx750m"

25+ - GHCVER=8.2 ESVER=6.1.3 STACK_YAML=stack.yaml ESFLAG=ES5 DLINK=https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch ES_JAVA_OPTS="-Xms500m -Xmx750m"

26install:

27install:

27 # stack

28 # stack

src/Database/V5/Bloodhound/Client.hs+10

@@ -54,6 +54,7 @@ module Database.V5.Bloodhound.Client

54 -- ** Searching

55 , searchAll

56 , searchByIndex

57+ , searchByIndices

57 , searchByType

58 , searchByType

58 , scanSearch

59 , scanSearch

59 , getInitialScroll

60 , getInitialScroll

@@ -995,6 +996,15 @@ searchByIndex :: MonadBH m => IndexName -> Search -> m Reply

995searchByIndex (IndexName indexName) = bindM2 dispatchSearch url . return

996searchByIndex (IndexName indexName) = bindM2 dispatchSearch url . return

996 where url = joinPath [indexName, "_search"]

997 where url = joinPath [indexName, "_search"]

997

998

999+-- | 'searchByIndices' is a variant of 'searchByIndex' that executes a

1000+-- 'Search' over many indices. This is much faster than using

1001+-- 'mapM' to 'searchByIndex' over a collection since it only

1002+-- causes a single HTTP request to be emitted.

1003+searchByIndices :: MonadBH m => NonEmpty IndexName -> Search -> m Reply

1004+searchByIndices ixs = bindM2 dispatchSearch url . return

1005+ where url = joinPath [renderedIxs, "_search"]

1006+ renderedIxs = T.intercalate (T.singleton ',') (map (\(IndexName t) -> t) (toList ixs))

1007+

998-- | 'searchByType', given a 'Search', 'IndexName', and 'MappingName', will perform that

1008-- | 'searchByType', given a 'Search', 'IndexName', and 'MappingName', will perform that

999-- search against a specific mapping within an index on an Elasticsearch server.

1009-- search against a specific mapping within an index on an Elasticsearch server.

1000--

1010--

src/Database/V5/Bloodhound/Internal/Analysis.hs+218−4

@@ -8,24 +8,29 @@ import Data.Aeson.Types (Parser)

8import qualified Data.Map.Strict as M

9import Data.Maybe (catMaybes)

10import Data.Text (Text)

11+import qualified Data.Text as T

13+import Database.V5.Bloodhound.Internal.Newtypes

12import Database.V5.Bloodhound.Internal.StringlyTyped

14import Database.V5.Bloodhound.Internal.StringlyTyped

14data Analysis = Analysis

16data Analysis = Analysis

15 { analysisAnalyzer :: M.Map Text AnalyzerDefinition

17 { analysisAnalyzer :: M.Map Text AnalyzerDefinition

16 , analysisTokenizer :: M.Map Text TokenizerDefinition

18 , analysisTokenizer :: M.Map Text TokenizerDefinition

19+ , analysisTokenFilter :: M.Map Text TokenFilterDefinition

17 } deriving (Eq, Show)

20 } deriving (Eq, Show)

19instance ToJSON Analysis where

22instance ToJSON Analysis where

20− toJSON (Analysis analyzer tokenizer) = object

23+ toJSON (Analysis analyzer tokenizer tokenFilter) = object

21 [ "analyzer" .= analyzer

24 [ "analyzer" .= analyzer

22 , "tokenizer" .= tokenizer

25 , "tokenizer" .= tokenizer

26+ , "filter" .= tokenFilter

23 ]

27 ]

25instance FromJSON Analysis where

29instance FromJSON Analysis where

26 parseJSON = withObject "Analysis" $ \m -> Analysis

30 parseJSON = withObject "Analysis" $ \m -> Analysis

27 <$> m .: "analyzer"

31 <$> m .: "analyzer"

28− <*> m .: "tokenizer"

32+ <*> m .:? "tokenizer" .!= M.empty

33+ <*> m .:? "filter" .!= M.empty

30newtype Tokenizer =

35newtype Tokenizer =

31 Tokenizer Text

36 Tokenizer Text

@@ -33,17 +38,19 @@ newtype Tokenizer =

34data AnalyzerDefinition = AnalyzerDefinition

39data AnalyzerDefinition = AnalyzerDefinition

35 { analyzerDefinitionTokenizer :: Maybe Tokenizer

40 { analyzerDefinitionTokenizer :: Maybe Tokenizer

41+ , analyzerDefinitionFilter :: [TokenFilter]

36 } deriving (Eq,Show)

42 } deriving (Eq,Show)

38instance ToJSON AnalyzerDefinition where

44instance ToJSON AnalyzerDefinition where

39− toJSON (AnalyzerDefinition tokenizer) = object $ catMaybes

45+ toJSON (AnalyzerDefinition tokenizer tokenFilter) = object $ catMaybes

40 [ fmap ("tokenizer" .=) tokenizer

46 [ fmap ("tokenizer" .=) tokenizer

47+ , Just $ "filter" .= tokenFilter

41 ]

48 ]

43instance FromJSON AnalyzerDefinition where

50instance FromJSON AnalyzerDefinition where

44 parseJSON = withObject "AnalyzerDefinition" $ \m -> AnalyzerDefinition

51 parseJSON = withObject "AnalyzerDefinition" $ \m -> AnalyzerDefinition

45 <$> m .:? "tokenizer"

52 <$> m .:? "tokenizer"

46−

53+ <*> m .:? "filter" .!= []

48data TokenizerDefinition

55data TokenizerDefinition

49 = TokenizerDefinitionNgram Ngram

56 = TokenizerDefinitionNgram Ngram

@@ -98,3 +105,210 @@ instance FromJSON TokenChar where

98 "punctuation" -> return TokenPunctuation

105 "punctuation" -> return TokenPunctuation

99 "symbol" -> return TokenSymbol

106 "symbol" -> return TokenSymbol

100 _ -> fail "invalid TokenChar"

107 _ -> fail "invalid TokenChar"

108+

109+-- | Token filters are used to create custom analyzers.

110+data TokenFilterDefinition

111+ = TokenFilterDefinitionLowercase (Maybe Language)

112+ | TokenFilterDefinitionUppercase (Maybe Language)

113+ | TokenFilterDefinitionApostrophe

114+ | TokenFilterDefinitionReverse

115+ | TokenFilterDefinitionSnowball Language

116+ | TokenFilterDefinitionShingle Shingle

117+ deriving (Eq, Show)

118+

119+instance ToJSON TokenFilterDefinition where

120+ toJSON x = case x of

121+ TokenFilterDefinitionLowercase mlang -> object $ catMaybes

122+ [ Just $ "type" .= ("lowercase" :: Text)

123+ , fmap (\lang -> "language" .= languageToText lang) mlang

124+ ]

125+ TokenFilterDefinitionUppercase mlang -> object $ catMaybes

126+ [ Just $ "type" .= ("uppercase" :: Text)

127+ , fmap (\lang -> "language" .= languageToText lang) mlang

128+ ]

129+ TokenFilterDefinitionApostrophe -> object

130+ [ "type" .= ("apostrophe" :: Text)

131+ ]

132+ TokenFilterDefinitionReverse -> object

133+ [ "type" .= ("reverse" :: Text)

134+ ]

135+ TokenFilterDefinitionSnowball lang -> object

136+ [ "type" .= ("snowball" :: Text)

137+ , "language" .= languageToText lang

138+ ]

139+ TokenFilterDefinitionShingle s -> object

140+ [ "type" .= ("shingle" :: Text)

141+ , "max_shingle_size" .= shingleMaxSize s

142+ , "min_shingle_size" .= shingleMinSize s

143+ , "output_unigrams" .= shingleOutputUnigrams s

144+ , "output_unigrams_if_no_shingles" .= shingleOutputUnigramsIfNoShingles s

145+ , "token_separator" .= shingleTokenSeparator s

146+ , "filler_token" .= shingleFillerToken s

147+ ]

148+

149+instance FromJSON TokenFilterDefinition where

150+ parseJSON = withObject "TokenFilterDefinition" $ \m -> do

151+ t <- m .: "type"

152+ case (t :: Text) of

153+ "reverse" -> return TokenFilterDefinitionReverse

154+ "apostrophe" -> return TokenFilterDefinitionApostrophe

155+ "lowercase" -> TokenFilterDefinitionLowercase

156+ <$> m .:? "language"

157+ "uppercase" -> TokenFilterDefinitionUppercase

158+ <$> m .:? "language"

159+ "snowball" -> TokenFilterDefinitionSnowball

160+ <$> m .: "language"

161+ "shingle" -> fmap TokenFilterDefinitionShingle $ Shingle

162+ <$> (fmap.fmap) unStringlyTypedInt (m .:? "max_shingle_size") .!= 2

163+ <*> (fmap.fmap) unStringlyTypedInt (m .:? "min_shingle_size") .!= 2

164+ <*> (fmap.fmap) unStringlyTypedBool (m .:? "output_unigrams") .!= True

165+ <*> (fmap.fmap) unStringlyTypedBool (m .:? "output_unigrams_if_no_shingles") .!= False

166+ <*> m .:? "token_separator" .!= " "

167+ <*> m .:? "filler_token" .!= "_"

168+ _ -> fail ("unrecognized token filter type: " ++ T.unpack t)

169+

170+-- | The set of languages that can be passed to various analyzers,

171+-- filters, etc. in ElasticSearch. Most data types in this module

172+-- that have a 'Language' field are actually only actually to

173+-- handle a subset of these languages. Consult the official

174+-- ElasticSearch documentation to see what is actually supported.

175+data Language

176+ = Arabic

177+ | Armenian

178+ | Basque

179+ | Bengali

180+ | Brazilian

181+ | Bulgarian

182+ | Catalan

183+ | Cjk

184+ | Czech

185+ | Danish

186+ | Dutch

187+ | English

188+ | Finnish

189+ | French

190+ | Galician

191+ | German

192+ | German2

193+ | Greek

194+ | Hindi

195+ | Hungarian

196+ | Indonesian

197+ | Irish

198+ | Italian

199+ | Kp

200+ | Latvian

201+ | Lithuanian

202+ | Lovins

203+ | Norwegian

204+ | Persian

205+ | Porter

206+ | Portuguese

207+ | Romanian

208+ | Russian

209+ | Sorani

210+ | Spanish

211+ | Swedish

212+ | Thai

213+ | Turkish

214+ deriving (Eq, Show)

215+

216+instance ToJSON Language where

217+ toJSON = Data.Aeson.String . languageToText

218+

219+instance FromJSON Language where

220+ parseJSON = withText "Language" $ \t -> case languageFromText t of

221+ Nothing -> fail "not a supported ElasticSearch language"

222+ Just lang -> return lang

223+

224+languageToText :: Language -> Text

225+languageToText x = case x of

226+ Arabic -> "arabic"

227+ Armenian -> "armenian"

228+ Basque -> "basque"

229+ Bengali -> "bengali"

230+ Brazilian -> "brazilian"

231+ Bulgarian -> "bulgarian"

232+ Catalan -> "catalan"

233+ Cjk -> "cjk"

234+ Czech -> "czech"

235+ Danish -> "danish"

236+ Dutch -> "dutch"

237+ English -> "english"

238+ Finnish -> "finnish"

239+ French -> "french"

240+ Galician -> "galician"

241+ German -> "german"

242+ German2 -> "german2"

243+ Greek -> "greek"

244+ Hindi -> "hindi"

245+ Hungarian -> "hungarian"

246+ Indonesian -> "indonesian"

247+ Irish -> "irish"

248+ Italian -> "italian"

249+ Kp -> "kp"

250+ Latvian -> "latvian"

251+ Lithuanian -> "lithuanian"

252+ Lovins -> "lovins"

253+ Norwegian -> "norwegian"

254+ Persian -> "persian"

255+ Porter -> "porter"

256+ Portuguese -> "portuguese"

257+ Romanian -> "romanian"

258+ Russian -> "russian"

259+ Sorani -> "sorani"

260+ Spanish -> "spanish"

261+ Swedish -> "swedish"

262+ Thai -> "thai"

263+ Turkish -> "turkish"

264+

265+languageFromText :: Text -> Maybe Language

266+languageFromText x = case x of

267+ "arabic" -> Just Arabic

268+ "armenian" -> Just Armenian

269+ "basque" -> Just Basque

270+ "bengali" -> Just Bengali

271+ "brazilian" -> Just Brazilian

272+ "bulgarian" -> Just Bulgarian

273+ "catalan" -> Just Catalan

274+ "cjk" -> Just Cjk

275+ "czech" -> Just Czech

276+ "danish" -> Just Danish

277+ "dutch" -> Just Dutch

278+ "english" -> Just English

279+ "finnish" -> Just Finnish

280+ "french" -> Just French

281+ "galician" -> Just Galician

282+ "german" -> Just German

283+ "german2" -> Just German2

284+ "greek" -> Just Greek

285+ "hindi" -> Just Hindi

286+ "hungarian" -> Just Hungarian

287+ "indonesian" -> Just Indonesian

288+ "irish" -> Just Irish

289+ "italian" -> Just Italian

290+ "kp" -> Just Kp

291+ "latvian" -> Just Latvian

292+ "lithuanian" -> Just Lithuanian

293+ "lovins" -> Just Lovins

294+ "norwegian" -> Just Norwegian

295+ "persian" -> Just Persian

296+ "porter" -> Just Porter

297+ "portuguese" -> Just Portuguese

298+ "romanian" -> Just Romanian

299+ "russian" -> Just Russian

300+ "sorani" -> Just Sorani

301+ "spanish" -> Just Spanish

302+ "swedish" -> Just Swedish

303+ "thai" -> Just Thai

304+ "turkish" -> Just Turkish

305+ _ -> Nothing

306+

307+data Shingle = Shingle

308+ { shingleMaxSize :: Int

309+ , shingleMinSize :: Int

310+ , shingleOutputUnigrams :: Bool

311+ , shingleOutputUnigramsIfNoShingles :: Bool

312+ , shingleTokenSeparator :: Text

313+ , shingleFillerToken :: Text

314+ } deriving (Eq, Show)

src/Database/V5/Bloodhound/Internal/Newtypes.hs+3

@@ -196,3 +196,6 @@ instance FromJSON MS where

196 parseJSON = withScientific "MS" (return . MS . parse)

197 where

198 parse n = fromInteger ((truncate n) * 1000)

199+

200+newtype TokenFilter =

201+ TokenFilter Text deriving (Eq, Show, FromJSON, ToJSON)

src/Database/V5/Bloodhound/Internal/StringlyTyped.hs+9−1

@@ -26,8 +26,16 @@ instance FromJSON StringlyTypedInt where

26 . parseJSON

27 . unStringlyTypeJSON

29+newtype StringlyTypedBool = StringlyTypedBool { unStringlyTypedBool :: Bool }

30+

31+instance FromJSON StringlyTypedBool where

32+ parseJSON =

33+ fmap StringlyTypedBool

34+ . parseJSON

35+ . unStringlyTypeJSON

36+

29-- | For some reason in several settings APIs, all leaf values get returned

37-- | For some reason in several settings APIs, all leaf values get returned

30−-- as strings. This function attepmts to recover from this for all

38+-- as strings. This function attempts to recover from this for all

31-- non-recursive JSON types. If nothing can be done, the value is left alone.

39-- non-recursive JSON types. If nothing can be done, the value is left alone.

32unStringlyTypeJSON :: Value -> Value

40unStringlyTypeJSON :: Value -> Value

33unStringlyTypeJSON (String "true") =

41unStringlyTypeJSON (String "true") =

src/Database/V5/Bloodhound/Types.hs+4−1

@@ -223,6 +223,7 @@ module Database.V5.Bloodhound.Types

223 , CutoffFrequency(..)

224 , Analyzer(..)

225 , Tokenizer(..)

226+ , TokenFilter(..)

226 , MaxExpansions(..)

227 , MaxExpansions(..)

227 , Lenient(..)

228 , Lenient(..)

228 , MatchQueryType(..)

229 , MatchQueryType(..)

@@ -404,8 +405,11 @@ module Database.V5.Bloodhound.Types

404 , Analysis(..)

405 , Analysis(..)

405 , AnalyzerDefinition(..)

406 , AnalyzerDefinition(..)

406 , TokenizerDefinition(..)

407 , TokenizerDefinition(..)

408+ , TokenFilterDefinition(..)

407 , Ngram(..)

409 , Ngram(..)

408 , TokenChar(..)

410 , TokenChar(..)

411+ , Shingle(..)

412+ , Language(..)

409 ) where

413 ) where

410

414

411import Bloodhound.Import

415import Bloodhound.Import

@@ -1575,7 +1579,6 @@ instance FromJSON DocVersion where

1575 i <- parseJSON v

1579 i <- parseJSON v

1576 maybe (fail "DocVersion out of range") return $ mkDocVersion i

1580 maybe (fail "DocVersion out of range") return $ mkDocVersion i

1577

1581

1578−

1579data Suggest = Suggest { suggestText :: Text

1582data Suggest = Suggest { suggestText :: Text

1580 , suggestName :: Text

1583 , suggestName :: Text

1581 , suggestType :: SuggestType

1584 , suggestType :: SuggestType

tests/V5/tests.hs+29−1

@@ -925,6 +925,14 @@ makeArbitrary ''Ngram

925instance Arbitrary Ngram where arbitrary = arbitraryNgram

926makeArbitrary ''TokenizerDefinition

927instance Arbitrary TokenizerDefinition where arbitrary = arbitraryTokenizerDefinition

928+makeArbitrary ''TokenFilter

929+instance Arbitrary TokenFilter where arbitrary = arbitraryTokenFilter

930+makeArbitrary ''TokenFilterDefinition

931+instance Arbitrary TokenFilterDefinition where arbitrary = arbitraryTokenFilterDefinition

932+makeArbitrary ''Language

933+instance Arbitrary Language where arbitrary = arbitraryLanguage

934+makeArbitrary ''Shingle

935+instance Arbitrary Shingle where arbitrary = arbitraryShingle

928makeArbitrary ''AnalyzerDefinition

936makeArbitrary ''AnalyzerDefinition

929instance Arbitrary AnalyzerDefinition where arbitrary = arbitraryAnalyzerDefinition

937instance Arbitrary AnalyzerDefinition where arbitrary = arbitraryAnalyzerDefinition

930makeArbitrary ''Analysis

938makeArbitrary ''Analysis

@@ -1023,6 +1031,7 @@ instance Arbitrary UpdatableIndexSetting' where

1023 NE.fromList . L.nubBy sameAttrName . NE.toList

1031 NE.fromList . L.nubBy sameAttrName . NE.toList

1024 sameAttrName a b =

1032 sameAttrName a b =

1025 nodeAttrFilterName a == nodeAttrFilterName b

1033 nodeAttrFilterName a == nodeAttrFilterName b

1034+ -- shrink (UpdatableIndexSetting' x) = map UpdatableIndexSetting' (shrink x)

1026

1035

1027-- | Due to the way nodeattrfilters get serialized here, they may come

1036-- | Due to the way nodeattrfilters get serialized here, they may come

1028-- out in a different order, but they are morally equivalent

1037-- out in a different order, but they are morally equivalent

@@ -1773,12 +1782,31 @@ main = hspec $ do

1773 it "accepts customer analyzers" $ when' (atleast es50) $ withTestEnv $ do

1782 it "accepts customer analyzers" $ when' (atleast es50) $ withTestEnv $ do

1774 _ <- deleteExampleIndex

1783 _ <- deleteExampleIndex

1775 let analysis = Analysis

1784 let analysis = Analysis

1785+ (M.singleton "ex_analyzer"

1786+ ( AnalyzerDefinition

1776− (M.singleton "ex_analyzer" (AnalyzerDefinition (Just (Tokenizer "ex_tokenizer"))))

1787+ (Just (Tokenizer "ex_tokenizer"))

1788+ (map TokenFilter

1789+ [ "ex_filter_lowercase","ex_filter_uppercase","ex_filter_apostrophe"

1790+ , "ex_filter_reverse","ex_filter_snowball"

1791+ , "ex_filter_shingle"

1792+ ]

1793+ )

1794+ )

1795+ )

1777 (M.singleton "ex_tokenizer"

1796 (M.singleton "ex_tokenizer"

1778 ( TokenizerDefinitionNgram

1797 ( TokenizerDefinitionNgram

1779 ( Ngram 3 4 [TokenLetter,TokenDigit])

1798 ( Ngram 3 4 [TokenLetter,TokenDigit])

1780 )

1799 )

1781 )

1800 )

1801+ (M.fromList

1802+ [ ("ex_filter_lowercase",TokenFilterDefinitionLowercase (Just Greek))

1803+ , ("ex_filter_uppercase",TokenFilterDefinitionUppercase Nothing)

1804+ , ("ex_filter_apostrophe",TokenFilterDefinitionApostrophe)

1805+ , ("ex_filter_reverse",TokenFilterDefinitionReverse)

1806+ , ("ex_filter_snowball",TokenFilterDefinitionSnowball English)

1807+ , ("ex_filter_shingle",TokenFilterDefinitionShingle (Shingle 3 3 True False " " "_"))

1808+ ]

1809+ )

1782 updates = [AnalysisSetting analysis]

1810 updates = [AnalysisSetting analysis]

1783 createResp <- createIndexWith (updates ++ [NumberOfReplicas (ReplicaCount 0)]) 1 testIndex

1811 createResp <- createIndexWith (updates ++ [NumberOfReplicas (ReplicaCount 0)]) 1 testIndex

1784 liftIO $ validateStatus createResp 200

1812 liftIO $ validateStatus createResp 200