2ddaedd · typeable/bloodhound

git.iko.soy/typeable/bloodhound

on master help sign in

Merged #230

2ddaeddChris Allen9 years ago

changelog.md+10−2

compare with master

unifiedsplit

@@ -1,8 +1,10 @@

10.16.0.0

2========

3- @bitemyapp

4− - Reorganized modules internally, ripped out Generic,

4+ - Reorganized V5 modules internally, ripped out Generic,

5− rewrote part of the test suite

5+ rewrote part of the test suite.

6+- @mxavier

7+ - Reorganized V1 modules, mirroring the work done on V5.

6- @andrewthad

8- @andrewthad

7 - Added support for autogenerated elasticsearch ids in the bulk API

9 - Added support for autogenerated elasticsearch ids in the bulk API

8 - Added support for token filters

10 - Added support for token filters

@@ -11,6 +13,12 @@

11 - Added support for scripts fields and function score queries

13 - Added support for scripts fields and function score queries

12- @bermanjosh

14- @bermanjosh

13 - Added support for direct generators

15 - Added support for direct generators

16+- @rvl

17+ - Add [character filters][] to the `AnalyzerDefinition` type.

18+ - Add the "stemmer" and "stop" [token filters][] to `TokenFilterDefinition`.

19+

20+[Character Filters]: https://www.elastic.co/guide/en/elasticsearch/reference/5.6/analysis-charfilters.html

21+[Token Filters]: https://www.elastic.co/guide/en/elasticsearch/reference/5.6/analysis-tokenfilters.html

150.15.0.2

230.15.0.2

16========

24========

src/Database/V5/Bloodhound/Internal/Analysis.hs+70−2

@@ -6,6 +6,7 @@ module Database.V5.Bloodhound.Internal.Analysis where

6import Bloodhound.Import

8import qualified Data.Map.Strict as M

9+import Data.String

9import qualified Data.Text as T

10import qualified Data.Text as T

11import Database.V5.Bloodhound.Internal.Newtypes

12import Database.V5.Bloodhound.Internal.Newtypes

@@ -15,13 +16,15 @@ data Analysis = Analysis

15 { analysisAnalyzer :: M.Map Text AnalyzerDefinition

16 { analysisAnalyzer :: M.Map Text AnalyzerDefinition

16 , analysisTokenizer :: M.Map Text TokenizerDefinition

17 , analysisTokenizer :: M.Map Text TokenizerDefinition

17 , analysisTokenFilter :: M.Map Text TokenFilterDefinition

18 , analysisTokenFilter :: M.Map Text TokenFilterDefinition

19+ , analysisCharFilter :: M.Map Text CharFilterDefinition

18 } deriving (Eq, Show)

20 } deriving (Eq, Show)

20instance ToJSON Analysis where

22instance ToJSON Analysis where

21− toJSON (Analysis analyzer tokenizer tokenFilter) = object

23+ toJSON (Analysis analyzer tokenizer tokenFilter charFilter) = object

22 [ "analyzer" .= analyzer

24 [ "analyzer" .= analyzer

23 , "tokenizer" .= tokenizer

25 , "tokenizer" .= tokenizer

24 , "filter" .= tokenFilter

26 , "filter" .= tokenFilter

27+ , "char_filter" .= charFilter

25 ]

28 ]

27instance FromJSON Analysis where

30instance FromJSON Analysis where

@@ -29,6 +32,7 @@ instance FromJSON Analysis where

29 <$> m .: "analyzer"

32 <$> m .: "analyzer"

30 <*> m .:? "tokenizer" .!= M.empty

33 <*> m .:? "tokenizer" .!= M.empty

31 <*> m .:? "filter" .!= M.empty

34 <*> m .:? "filter" .!= M.empty

35+ <*> m .:? "char_filter" .!= M.empty

33newtype Tokenizer =

37newtype Tokenizer =

34 Tokenizer Text

38 Tokenizer Text

@@ -37,18 +41,57 @@ newtype Tokenizer =

37data AnalyzerDefinition = AnalyzerDefinition

41data AnalyzerDefinition = AnalyzerDefinition

38 { analyzerDefinitionTokenizer :: Maybe Tokenizer

42 { analyzerDefinitionTokenizer :: Maybe Tokenizer

39 , analyzerDefinitionFilter :: [TokenFilter]

43 , analyzerDefinitionFilter :: [TokenFilter]

44+ , analyzerDefinitionCharFilter :: [CharFilter]

40 } deriving (Eq,Show)

45 } deriving (Eq,Show)

42instance ToJSON AnalyzerDefinition where

47instance ToJSON AnalyzerDefinition where

43− toJSON (AnalyzerDefinition tokenizer tokenFilter) = object $ catMaybes

48+ toJSON (AnalyzerDefinition tokenizer tokenFilter charFilter) =

49+ object $ catMaybes

44 [ fmap ("tokenizer" .=) tokenizer

50 [ fmap ("tokenizer" .=) tokenizer

45 , Just $ "filter" .= tokenFilter

51 , Just $ "filter" .= tokenFilter

52+ , Just $ "char_filter" .= charFilter

46 ]

53 ]

48instance FromJSON AnalyzerDefinition where

55instance FromJSON AnalyzerDefinition where

49 parseJSON = withObject "AnalyzerDefinition" $ \m -> AnalyzerDefinition

56 parseJSON = withObject "AnalyzerDefinition" $ \m -> AnalyzerDefinition

50 <$> m .:? "tokenizer"

57 <$> m .:? "tokenizer"

51 <*> m .:? "filter" .!= []

58 <*> m .:? "filter" .!= []

59+ <*> m .:? "char_filter" .!= []

60+

61+-- | Character filters are used to preprocess the stream of characters

62+-- before it is passed to the tokenizer.

63+data CharFilterDefinition

64+ = CharFilterDefinitionMapping (M.Map Text Text)

65+ | CharFilterDefinitionPatternReplace

66+ { charFilterDefinitionPatternReplacePattern :: Text

67+ , charFilterDefinitionPatternReplaceReplacement :: Text

68+ , charFilterDefinitionPatternReplaceFlags :: Maybe Text

69+ }

70+ deriving (Eq, Show)

71+

72+instance ToJSON CharFilterDefinition where

73+ toJSON (CharFilterDefinitionMapping ms) = object

74+ [ "type" .= ("mapping" :: Text)

75+ , "mappings" .= [a <> " => " <> b | (a, b) <- M.toList ms] ]

76+ toJSON (CharFilterDefinitionPatternReplace pat repl flags) = object $

77+ [ "type" .= ("pattern_replace" :: Text)

78+ , "pattern" .= pat

79+ , "replacement" .= repl

80+ ] ++ maybe [] (\f -> ["flags" .= f]) flags

81+

82+instance FromJSON CharFilterDefinition where

83+ parseJSON = withObject "CharFilterDefinition" $ \m -> do

84+ t <- m .: "type"

85+ case (t :: Text) of

86+ "mapping" -> CharFilterDefinitionMapping . M.fromList <$> ms

87+ where

88+ ms = m .: "mappings" >>= mapM parseMapping

89+ parseMapping kv = case T.splitOn "=>" kv of

90+ (k:vs) -> pure (T.strip k, T.strip $ T.concat vs)

91+ _ -> fail "mapping is not of the format key => value"

92+ "pattern_replace" -> CharFilterDefinitionPatternReplace

93+ <$> m .: "pattern" <*> m .: "replacement" <*> m .:? "flags"

94+ _ -> fail ("unrecognized character filter type: " ++ T.unpack t)

53newtype TokenizerDefinition =

96newtype TokenizerDefinition =

54 TokenizerDefinitionNgram Ngram

97 TokenizerDefinitionNgram Ngram

@@ -112,6 +155,8 @@ data TokenFilterDefinition

112 | TokenFilterDefinitionReverse

155 | TokenFilterDefinitionReverse

113 | TokenFilterDefinitionSnowball Language

156 | TokenFilterDefinitionSnowball Language

114 | TokenFilterDefinitionShingle Shingle

157 | TokenFilterDefinitionShingle Shingle

158+ | TokenFilterDefinitionStemmer Language

159+ | TokenFilterDefinitionStop (Either Language [StopWord])

115 deriving (Eq, Show)

160 deriving (Eq, Show)

116

161

117instance ToJSON TokenFilterDefinition where

162instance ToJSON TokenFilterDefinition where

@@ -143,6 +188,16 @@ instance ToJSON TokenFilterDefinition where

143 , "token_separator" .= shingleTokenSeparator s

188 , "token_separator" .= shingleTokenSeparator s

144 , "filler_token" .= shingleFillerToken s

189 , "filler_token" .= shingleFillerToken s

145 ]

190 ]

191+ TokenFilterDefinitionStemmer lang -> object

192+ [ "type" .= ("stemmer" :: Text)

193+ , "language" .= languageToText lang

194+ ]

195+ TokenFilterDefinitionStop stop -> object

196+ [ "type" .= ("stop" :: Text)

197+ , "stopwords" .= case stop of

198+ Left lang -> String $ "_" <> languageToText lang <> "_"

199+ Right stops -> toJSON stops

200+ ]

146

201

147instance FromJSON TokenFilterDefinition where

202instance FromJSON TokenFilterDefinition where

148 parseJSON = withObject "TokenFilterDefinition" $ \m -> do

203 parseJSON = withObject "TokenFilterDefinition" $ \m -> do

@@ -163,6 +218,19 @@ instance FromJSON TokenFilterDefinition where

163 <*> (fmap.fmap) unStringlyTypedBool (m .:? "output_unigrams_if_no_shingles") .!= False

218 <*> (fmap.fmap) unStringlyTypedBool (m .:? "output_unigrams_if_no_shingles") .!= False

164 <*> m .:? "token_separator" .!= " "

219 <*> m .:? "token_separator" .!= " "

165 <*> m .:? "filler_token" .!= "_"

220 <*> m .:? "filler_token" .!= "_"

221+ "stemmer" -> TokenFilterDefinitionStemmer

222+ <$> m .: "language"

223+ "stop" -> do

224+ stop <- m .: "stopwords"

225+ stop' <- case stop of

226+ String lang ->

227+ fmap Left

228+ . parseJSON

229+ . String

230+ . T.drop 1

231+ . T.dropEnd 1 $ lang

232+ _ -> Right <$> parseJSON stop

233+ return (TokenFilterDefinitionStop stop')

166 _ -> fail ("unrecognized token filter type: " ++ T.unpack t)

234 _ -> fail ("unrecognized token filter type: " ++ T.unpack t)

167

235

168-- | The set of languages that can be passed to various analyzers,

236-- | The set of languages that can be passed to various analyzers,

src/Database/V5/Bloodhound/Internal/Newtypes.hs+6−1

@@ -231,4 +231,9 @@ instance FromJSON MS where

231 parse n = fromInteger (truncate n * 1000)

232

233newtype TokenFilter =

234+ TokenFilter Text

235+ deriving (Eq, Show, FromJSON, ToJSON)

236+

237+newtype CharFilter =

238+ CharFilter Text

234− TokenFilter Text deriving (Eq, Show, FromJSON, ToJSON)

239+ deriving (Eq, Show, FromJSON, ToJSON)

src/Database/V5/Bloodhound/Internal/Suggest.hs+6−1

@@ -262,4 +262,9 @@ instance FromJSON DirectGenerators where

262 <*> o .:? "post_filter"

263

264mkDirectGenerators :: FieldName -> DirectGenerators

265+mkDirectGenerators fn =

266+ DirectGenerators

267+ fn

268+ Nothing

269+ DirectGeneratorSuggestModeMissing

265−mkDirectGenerators fn = DirectGenerators fn Nothing DirectGeneratorSuggestModeMissing Nothing Nothing Nothing Nothing Nothing Nothing Nothing Nothing

270+ Nothing Nothing Nothing Nothing Nothing Nothing Nothing Nothing

src/Database/V5/Bloodhound/Types.hs+2

@@ -224,6 +224,7 @@ module Database.V5.Bloodhound.Types

224 , Analyzer(..)

225 , Tokenizer(..)

226 , TokenFilter(..)

227+ , CharFilter(..)

227 , MaxExpansions(..)

228 , MaxExpansions(..)

228 , Lenient(..)

229 , Lenient(..)

229 , MatchQueryType(..)

230 , MatchQueryType(..)

@@ -406,6 +407,7 @@ module Database.V5.Bloodhound.Types

406 , AnalyzerDefinition(..)

407 , AnalyzerDefinition(..)

407 , TokenizerDefinition(..)

408 , TokenizerDefinition(..)

408 , TokenFilterDefinition(..)

409 , TokenFilterDefinition(..)

410+ , CharFilterDefinition(..)

409 , Ngram(..)

411 , Ngram(..)

410 , TokenChar(..)

412 , TokenChar(..)

411 , Shingle(..)

413 , Shingle(..)

tests/V5/Test/Generators.hs+18−1

@@ -435,8 +435,25 @@ makeArbitrary ''Language

435instance Arbitrary Language where arbitrary = arbitraryLanguage

436makeArbitrary ''Shingle

437instance Arbitrary Shingle where arbitrary = arbitraryShingle

438+

439+makeArbitrary ''CharFilter

440+instance Arbitrary CharFilter where arbitrary = arbitraryCharFilter

438makeArbitrary ''AnalyzerDefinition

441makeArbitrary ''AnalyzerDefinition

439instance Arbitrary AnalyzerDefinition where arbitrary = arbitraryAnalyzerDefinition

442instance Arbitrary AnalyzerDefinition where arbitrary = arbitraryAnalyzerDefinition

443+

444+-- TODO: This should have a proper generator that doesn't

445+-- create garbage that has to be filtered out.

446+instance Arbitrary CharFilterDefinition where

447+ arbitrary =

448+ oneof [ CharFilterDefinitionMapping

449+ . chomp <$> arbitrary

450+ , CharFilterDefinitionPatternReplace

451+ <$> arbitrary <*> arbitrary <*> arbitrary

452+ ]

453+ where chomp =

454+ M.map T.strip

455+ . M.mapKeys (T.replace "=>" "" . T.strip)

456+

440makeArbitrary ''Analysis

457makeArbitrary ''Analysis

441instance Arbitrary Analysis where arbitrary = arbitraryAnalysis

458instance Arbitrary Analysis where arbitrary = arbitraryAnalysis

442makeArbitrary ''Tokenizer

459makeArbitrary ''Tokenizer

@@ -533,4 +550,4 @@ instance Arbitrary UpdatableIndexSetting' where

533 NE.fromList . L.nubBy sameAttrName . NE.toList

550 NE.fromList . L.nubBy sameAttrName . NE.toList

534 sameAttrName a b =

551 sameAttrName a b =

535 nodeAttrFilterName a == nodeAttrFilterName b

552 nodeAttrFilterName a == nodeAttrFilterName b

536− -- shrink (UpdatableIndexSetting' x) = map UpdatableIndexSetting' (shrink x)

553+ shrink (UpdatableIndexSetting' x) = map UpdatableIndexSetting' (shrink x)

tests/V5/Test/Indices.hs+15−1

@@ -99,6 +99,9 @@ spec = do

99 , "ex_filter_shingle"

100 ]

101 )

102+ (map CharFilter

103+ ["html_strip", "ex_mapping", "ex_pattern_replace"]

104+ )

102 )

105 )

103 )

106 )

104 (M.singleton "ex_tokenizer"

107 (M.singleton "ex_tokenizer"

@@ -113,7 +116,18 @@ spec = do

113 , ("ex_filter_reverse",TokenFilterDefinitionReverse)

116 , ("ex_filter_reverse",TokenFilterDefinitionReverse)

114 , ("ex_filter_snowball",TokenFilterDefinitionSnowball English)

117 , ("ex_filter_snowball",TokenFilterDefinitionSnowball English)

115 , ("ex_filter_shingle",TokenFilterDefinitionShingle (Shingle 3 3 True False " " "_"))

118 , ("ex_filter_shingle",TokenFilterDefinitionShingle (Shingle 3 3 True False " " "_"))

119+ , ("ex_filter_stemmer",TokenFilterDefinitionStemmer German)

120+ , ("ex_filter_stop1", TokenFilterDefinitionStop (Left French))

121+ , ("ex_filter_stop2",

122+ TokenFilterDefinitionStop

123+ (Right

124+ $ map StopWord ["a", "is", "the"]))

125+ ]

126+ )

127+ (M.fromList

128+ [ ("ex_mapping", CharFilterDefinitionMapping (M.singleton "١" "1"))

129+ , ("ex_pattern_replace", CharFilterDefinitionPatternReplace "(\\d+)-(?=\\d)" "$1_" Nothing)

116− ]

130+ ]

117 )

131 )

118 updates = [AnalysisSetting analysis]

132 updates = [AnalysisSetting analysis]

119 createResp <- createIndexWith (updates ++ [NumberOfReplicas (ReplicaCount 0)]) 1 testIndex

133 createResp <- createIndexWith (updates ++ [NumberOfReplicas (ReplicaCount 0)]) 1 testIndex