e972510 · typeable/bloodhound

git.iko.soy/typeable/bloodhound

on master help sign in

Add suggester to V1

e972510Josh Berman9 years ago

src/Database/V1/Bloodhound/Client.hs+3−3

compare with master

unifiedsplit

@@ -861,17 +861,17 @@ scanSearch indexName mappingName search = do

861-- syntax if you want to add things like aggregations or highlights while still using

862-- this helper function.

863mkSearch :: Maybe Query -> Maybe Filter -> Search

864−mkSearch query filter = Search query filter Nothing Nothing Nothing False (From 0) (Size 10) SearchTypeQueryThenFetch Nothing Nothing

864+mkSearch query filter = Search query filter Nothing Nothing Nothing False (From 0) (Size 10) SearchTypeQueryThenFetch Nothing Nothing Nothing

865

866-- | 'mkAggregateSearch' is a helper function that defaults everything in a 'Search' except for

867-- the 'Query' and the 'Aggregation'.

868mkAggregateSearch :: Maybe Query -> Aggregations -> Search

869−mkAggregateSearch query mkSearchAggs = Search query Nothing Nothing (Just mkSearchAggs) Nothing False (From 0) (Size 0) SearchTypeQueryThenFetch Nothing Nothing

869+mkAggregateSearch query mkSearchAggs = Search query Nothing Nothing (Just mkSearchAggs) Nothing False (From 0) (Size 0) SearchTypeQueryThenFetch Nothing Nothing Nothing

870

871-- | 'mkHighlightSearch' is a helper function that defaults everything in a 'Search' except for

872-- the 'Query' and the 'Aggregation'.

873mkHighlightSearch :: Maybe Query -> Highlights -> Search

874−mkHighlightSearch query searchHighlights = Search query Nothing Nothing Nothing (Just searchHighlights) False (From 0) (Size 10) SearchTypeQueryThenFetch Nothing Nothing

874+mkHighlightSearch query searchHighlights = Search query Nothing Nothing Nothing (Just searchHighlights) False (From 0) (Size 10) SearchTypeQueryThenFetch Nothing Nothing Nothing

875

876-- | 'pageSearch' is a helper function that takes a search and assigns the from

877-- and size fields for the search. The from parameter defines the offset

src/Database/V1/Bloodhound/Types.hs+197−7

@@ -324,6 +324,15 @@ module Database.V1.Bloodhound.Types

324 , rrGroupRefNum

325 , mkRRGroupRefNum

326 , RestoreIndexSettings(..)

327+ , Suggest(..)

328+ , SuggestType(..)

329+ , PhraseSuggester(..)

330+ , PhraseSuggesterHighlighter(..)

331+ , PhraseSuggesterCollate(..)

332+ , mkPhraseSuggester

333+ , SuggestOptions(..)

334+ , SuggestResponse(..)

335+ , NamedSuggestionResponse(..)

327

336

328 , Aggregation(..)

337 , Aggregation(..)

329 , Aggregations

338 , Aggregations

@@ -378,7 +387,8 @@ import Control.Monad.Writer (MonadWriter)

378import Data.Aeson

387import Data.Aeson

379import Data.Aeson.Types (Pair, Parser,

388import Data.Aeson.Types (Pair, Parser,

380 emptyObject,

389 emptyObject,

381− parseEither, parseMaybe)

390+ parseEither, parseMaybe,

391+ typeMismatch)

382import qualified Data.ByteString.Lazy.Char8 as L

392import qualified Data.ByteString.Lazy.Char8 as L

383import Data.Char

393import Data.Char

384import Data.Hashable (Hashable)

394import Data.Hashable (Hashable)

@@ -1093,7 +1103,7 @@ unpackId (DocId docId) = docId

1093

1103

1094type TrackSortScores = Bool

1104type TrackSortScores = Bool

1095newtype From = From Int deriving (Eq, Read, Show, Generic, ToJSON)

1105newtype From = From Int deriving (Eq, Read, Show, Generic, ToJSON)

1096−newtype Size = Size Int deriving (Eq, Read, Show, Generic, ToJSON)

1106+newtype Size = Size Int deriving (Eq, Read, Show, Generic, ToJSON, FromJSON)

1097

1107

1098data Search = Search { queryBody :: Maybe Query

1108data Search = Search { queryBody :: Maybe Query

1099 , filterBody :: Maybe Filter

1109 , filterBody :: Maybe Filter

@@ -1106,7 +1116,9 @@ data Search = Search { queryBody :: Maybe Query

1106 , size :: Size

1116 , size :: Size

1107 , searchType :: SearchType

1117 , searchType :: SearchType

1108 , fields :: Maybe [FieldName]

1118 , fields :: Maybe [FieldName]

1119+ , source :: Maybe Source

1120+ , suggestBody :: Maybe Suggest -- ^ Only one Suggestion request / response per Search is supported.

1109− , source :: Maybe Source } deriving (Eq, Read, Show, Generic, Typeable)

1121+ } deriving (Eq, Read, Show, Generic, Typeable)

1110

1122

1111data SearchType = SearchTypeQueryThenFetch

1123data SearchType = SearchTypeQueryThenFetch

1112 | SearchTypeDfsQueryThenFetch

1124 | SearchTypeDfsQueryThenFetch

@@ -1676,7 +1688,10 @@ data SearchResult a =

1676 , shards :: ShardResult

1688 , shards :: ShardResult

1677 , searchHits :: SearchHits a

1689 , searchHits :: SearchHits a

1678 , aggregations :: Maybe AggregationResults

1690 , aggregations :: Maybe AggregationResults

1691+ , scrollId :: Maybe ScrollId

1692+ , suggest :: Maybe NamedSuggestionResponse -- ^ Only one Suggestion request / response per Search is supported.

1693+ }

1679− , scrollId :: Maybe ScrollId } deriving (Eq, Read, Show, Generic, Typeable)

1694+ deriving (Eq, Read, Show, Generic, Typeable)

1680

1695

1681newtype ScrollId = ScrollId Text deriving (Eq, Read, Show, Generic, Ord, ToJSON, FromJSON)

1696newtype ScrollId = ScrollId Text deriving (Eq, Read, Show, Generic, Ord, ToJSON, FromJSON)

1682

1697

@@ -3452,7 +3467,7 @@ instance FromJSON SearchAliasRouting where

3452 where parse t = SearchAliasRouting <$> parseNEJSON (String <$> T.splitOn "," t)

3467 where parse t = SearchAliasRouting <$> parseNEJSON (String <$> T.splitOn "," t)

3453

3468

3454instance ToJSON Search where

3469instance ToJSON Search where

3455− toJSON (Search query sFilter sort searchAggs highlight sTrackSortScores sFrom sSize _ sFields sSource) =

3470+ toJSON (Search query sFilter sort searchAggs highlight sTrackSortScores sFrom sSize _ sFields sSource sSuggest) =

3456 omitNulls [ "query" .= query

3471 omitNulls [ "query" .= query

3457 , "filter" .= sFilter

3472 , "filter" .= sFilter

3458 , "sort" .= sort

3473 , "sort" .= sort

@@ -3462,7 +3477,8 @@ instance ToJSON Search where

3462 , "size" .= sSize

3477 , "size" .= sSize

3463 , "track_scores" .= sTrackSortScores

3478 , "track_scores" .= sTrackSortScores

3464 , "fields" .= sFields

3479 , "fields" .= sFields

3465− , "_source" .= sSource]

3480+ , "_source" .= sSource

3481+ , "suggest" .= sSuggest]

3466

3482

3467

3483

3468instance ToJSON Source where

3484instance ToJSON Source where

@@ -3812,7 +3828,8 @@ instance (FromJSON a) => FromJSON (SearchResult a) where

3812 v .: "_shards" <*>

3828 v .: "_shards" <*>

3813 v .: "hits" <*>

3829 v .: "hits" <*>

3814 v .:? "aggregations" <*>

3830 v .:? "aggregations" <*>

3815− v .:? "_scroll_id"

3831+ v .:? "_scroll_id" <*>

3832+ v .:? "suggest"

3816 parseJSON _ = empty

3833 parseJSON _ = empty

3817

3834

3818instance (FromJSON a) => FromJSON (SearchHits a) where

3835instance (FromJSON a) => FromJSON (SearchHits a) where

@@ -5317,3 +5334,176 @@ newtype MaybeNA a = MaybeNA { unMaybeNA :: Maybe a }

5317instance FromJSON a => FromJSON (MaybeNA a) where

5334instance FromJSON a => FromJSON (MaybeNA a) where

5318 parseJSON (String "NA") = pure $ MaybeNA Nothing

5335 parseJSON (String "NA") = pure $ MaybeNA Nothing

5319 parseJSON o = MaybeNA . Just <$> parseJSON o

5336 parseJSON o = MaybeNA . Just <$> parseJSON o

5337+

5338+

5339+data Suggest = Suggest { suggestText :: Text

5340+ , suggestName :: Text

5341+ , suggestType :: SuggestType

5342+ }

5343+ deriving (Show, Generic, Eq, Read)

5344+

5345+instance ToJSON Suggest where

5346+ toJSON Suggest{..} = object [ "text" .= suggestText

5347+ , suggestName .= suggestType

5348+ ]

5349+

5350+instance FromJSON Suggest where

5351+ parseJSON (Object o) = do

5352+ suggestText' <- o .: "text"

5353+ let dropTextList = HM.toList $ HM.filterWithKey (\x _ -> x /= "text") o

5354+ suggestName' <- case dropTextList of

5355+ [(x, _)] -> return x

5356+ _ -> fail "error parsing Suggest field name"

5357+ suggestType' <- o .: suggestName'

5358+ return $ Suggest suggestText' suggestName' suggestType'

5359+ parseJSON x = typeMismatch "Suggest" x

5360+

5361+data SuggestType = SuggestTypePhraseSuggester PhraseSuggester

5362+ deriving (Show, Generic, Eq, Read)

5363+

5364+instance ToJSON SuggestType where

5365+ toJSON (SuggestTypePhraseSuggester x) = object ["phrase" .= x]

5366+

5367+instance FromJSON SuggestType where

5368+ parseJSON = withObject "SuggestType" parse

5369+ where parse o = phraseSuggester `taggedWith` "phrase"

5370+ where taggedWith parser k = parser =<< o .: k

5371+ phraseSuggester = pure . SuggestTypePhraseSuggester

5372+

5373+data PhraseSuggester =

5374+ PhraseSuggester { phraseSuggesterField :: FieldName

5375+ , phraseSuggesterGramSize :: Maybe Int

5376+ , phraseSuggesterRealWordErrorLikelihood :: Maybe Int

5377+ , phraseSuggesterConfidence :: Maybe Int

5378+ , phraseSuggesterMaxErrors :: Maybe Int

5379+ , phraseSuggesterSeparator :: Maybe Text

5380+ , phraseSuggesterSize :: Maybe Size

5381+ , phraseSuggesterAnalyzer :: Maybe Analyzer

5382+ , phraseSuggesterShardSize :: Maybe Int

5383+ , phraseSuggesterHighlight :: Maybe PhraseSuggesterHighlighter

5384+ , phraseSuggesterCollate :: Maybe PhraseSuggesterCollate

5385+ }

5386+ deriving (Show, Generic, Eq, Read)

5387+

5388+instance ToJSON PhraseSuggester where

5389+ toJSON PhraseSuggester{..} = omitNulls [ "field" .= phraseSuggesterField

5390+ , "gram_size" .= phraseSuggesterGramSize

5391+ , "real_word_error_likelihood" .= phraseSuggesterRealWordErrorLikelihood

5392+ , "confidence" .= phraseSuggesterConfidence

5393+ , "max_errors" .= phraseSuggesterMaxErrors

5394+ , "separator" .= phraseSuggesterSeparator

5395+ , "size" .= phraseSuggesterSize

5396+ , "analyzer" .= phraseSuggesterAnalyzer

5397+ , "shard_size" .= phraseSuggesterShardSize

5398+ , "highlight" .= phraseSuggesterHighlight

5399+ , "collate" .= phraseSuggesterCollate

5400+ ]

5401+

5402+instance FromJSON PhraseSuggester where

5403+ parseJSON = withObject "PhraseSuggester" parse

5404+ where parse o = PhraseSuggester

5405+ <$> o .: "field"

5406+ <*> o .:? "gram_size"

5407+ <*> o .:? "real_word_error_likelihood"

5408+ <*> o .:? "confidence"

5409+ <*> o .:? "max_errors"

5410+ <*> o .:? "separator"

5411+ <*> o .:? "size"

5412+ <*> o .:? "analyzer"

5413+ <*> o .:? "shard_size"

5414+ <*> o .:? "highlight"

5415+ <*> o .:? "collate"

5416+

5417+mkPhraseSuggester :: FieldName -> PhraseSuggester

5418+mkPhraseSuggester fName =

5419+ PhraseSuggester fName Nothing Nothing Nothing Nothing Nothing Nothing

5420+ Nothing Nothing Nothing Nothing

5421+

5422+data PhraseSuggesterHighlighter =

5423+ PhraseSuggesterHighlighter { phraseSuggesterHighlighterPreTag :: Text

5424+ , phraseSuggesterHighlighterPostTag :: Text

5425+ }

5426+ deriving (Show, Generic, Eq, Read)

5427+

5428+instance ToJSON PhraseSuggesterHighlighter where

5429+ toJSON PhraseSuggesterHighlighter{..} =

5430+ object [ "pre_tag" .= phraseSuggesterHighlighterPreTag

5431+ , "post_tag" .= phraseSuggesterHighlighterPostTag

5432+ ]

5433+

5434+instance FromJSON PhraseSuggesterHighlighter where

5435+ parseJSON = withObject "PhraseSuggesterHighlighter" parse

5436+ where parse o = PhraseSuggesterHighlighter

5437+ <$> o .: "pre_tag"

5438+ <*> o .: "post_tag"

5439+

5440+data PhraseSuggesterCollate =

5441+ PhraseSuggesterCollate { phraseSuggesterCollateTemplateQuery :: TemplateQueryInline

5442+ , phraseSuggesterCollatePrune :: Bool

5443+ }

5444+ deriving (Show, Generic, Eq, Read)

5445+

5446+instance ToJSON PhraseSuggesterCollate where

5447+ toJSON PhraseSuggesterCollate{..} = object [ "query" .= object

5448+ [ "inline" .= (inline phraseSuggesterCollateTemplateQuery)

5449+ ]

5450+ , "params" .= (params phraseSuggesterCollateTemplateQuery)

5451+ , "prune" .= phraseSuggesterCollatePrune

5452+ ]

5453+

5454+instance FromJSON PhraseSuggesterCollate where

5455+ parseJSON (Object o) = do

5456+ query' <- o .: "query"

5457+ inline' <- query' .: "inline"

5458+ params' <- o .: "params"

5459+ prune' <- o .:? "prune" .!= False

5460+ return $ PhraseSuggesterCollate (TemplateQueryInline inline' params') prune'

5461+ parseJSON x = typeMismatch "PhraseSuggesterCollate" x

5462+

5463+data SuggestOptions =

5464+ SuggestOptions { suggestOptionsText :: Text

5465+ , suggestOptionsScore :: Double

5466+ , suggestOptionsFreq :: Maybe Int

5467+ , suggestOptionsHighlighted :: Maybe Text

5468+ }

5469+ deriving (Eq, Read, Show)

5470+

5471+instance FromJSON SuggestOptions where

5472+ parseJSON = withObject "SuggestOptions" parse

5473+ where parse o = SuggestOptions

5474+ <$> o .: "text"

5475+ <*> o .: "score"

5476+ <*> o .:? "freq"

5477+ <*> o .:? "highlighted"

5478+

5479+data SuggestResponse =

5480+ SuggestResponse { suggestResponseText :: Text

5481+ , suggestResponseOffset :: Int

5482+ , suggestResponseLength :: Int

5483+ , suggestResponseOptions :: [SuggestOptions]

5484+ }

5485+ deriving (Eq, Read, Show)

5486+

5487+instance FromJSON SuggestResponse where

5488+ parseJSON = withObject "SuggestResponse" parse

5489+ where parse o = SuggestResponse

5490+ <$> o .: "text"

5491+ <*> o .: "offset"

5492+ <*> o .: "length"

5493+ <*> o .: "options"

5494+

5495+data NamedSuggestionResponse =

5496+ NamedSuggestionResponse { nsrName :: Text

5497+ , nsrResponses :: [SuggestResponse]

5498+ }

5499+ deriving (Eq, Read, Show)

5500+

5501+instance FromJSON NamedSuggestionResponse where

5502+ parseJSON (Object o) = do

5503+ suggestionName' <- case HM.toList o of

5504+ [(x, _)] -> return x

5505+ _ -> fail "error parsing NamedSuggestionResponse name"

5506+ suggestionResponses' <- o .: suggestionName'

5507+ return $ NamedSuggestionResponse suggestionName' suggestionResponses'

5508+

5509+ parseJSON x = typeMismatch "NamedSuggestionResponse" x

tests/V1/tests.hs+30

@@ -598,6 +598,12 @@ instance ApproxEq BuildHash

598instance ApproxEq TemplateQueryKeyValuePairs where

599 (=~) = (==)

600instance ApproxEq TemplateQueryInline

601+instance ApproxEq Size

602+instance ApproxEq PhraseSuggesterHighlighter

603+instance ApproxEq PhraseSuggesterCollate

604+instance ApproxEq PhraseSuggester

605+instance ApproxEq SuggestType

606+instance ApproxEq Suggest

601

607

602-- | Due to the way nodeattrfilters get serialized here, they may come

608-- | Due to the way nodeattrfilters get serialized here, they may come

603-- out in a different order, but they are morally equivalent

609-- out in a different order, but they are morally equivalent

@@ -922,6 +928,12 @@ instance Arbitrary CompoundFormat where arbitrary = sopArbitrary; shrink = gener

922instance Arbitrary FsSnapshotRepo where arbitrary = sopArbitrary; shrink = genericShrink

928instance Arbitrary FsSnapshotRepo where arbitrary = sopArbitrary; shrink = genericShrink

923instance Arbitrary SnapshotRepoName where arbitrary = sopArbitrary; shrink = genericShrink

929instance Arbitrary SnapshotRepoName where arbitrary = sopArbitrary; shrink = genericShrink

924instance Arbitrary TemplateQueryInline where arbitrary = sopArbitrary; shrink = genericShrink

930instance Arbitrary TemplateQueryInline where arbitrary = sopArbitrary; shrink = genericShrink

931+instance Arbitrary PhraseSuggesterCollate where arbitrary = sopArbitrary; shrink = genericShrink

932+instance Arbitrary PhraseSuggesterHighlighter where arbitrary = sopArbitrary; shrink = genericShrink

933+instance Arbitrary Size where arbitrary = sopArbitrary; shrink = genericShrink

934+instance Arbitrary PhraseSuggester where arbitrary = sopArbitrary; shrink = genericShrink

935+instance Arbitrary SuggestType where arbitrary = sopArbitrary; shrink = genericShrink

936+instance Arbitrary Suggest where arbitrary = sopArbitrary; shrink = genericShrink

925

937

926newtype UpdatableIndexSetting' = UpdatableIndexSetting' UpdatableIndexSetting

938newtype UpdatableIndexSetting' = UpdatableIndexSetting' UpdatableIndexSetting

927 deriving (Show, Eq, ToJSON, FromJSON, ApproxEq, Typeable)

939 deriving (Show, Eq, ToJSON, FromJSON, ApproxEq, Typeable)

@@ -1157,6 +1169,7 @@ main = hspec $ do

1157 let search = Search Nothing

1169 let search = Search Nothing

1158 (Just IdentityFilter) (Just [sortSpec]) Nothing Nothing

1170 (Just IdentityFilter) (Just [sortSpec]) Nothing Nothing

1159 False (From 0) (Size 10) SearchTypeQueryThenFetch Nothing Nothing

1171 False (From 0) (Size 10) SearchTypeQueryThenFetch Nothing Nothing

1172+ Nothing

1160 result <- searchTweets search

1173 result <- searchTweets search

1161 let myTweet = grabFirst result

1174 let myTweet = grabFirst result

1162 liftIO $

1175 liftIO $

@@ -1785,6 +1798,22 @@ main = hspec $ do

1785 resp <- optimizeIndex (IndexList (testIndex :| [])) defaultIndexOptimizationSettings

1798 resp <- optimizeIndex (IndexList (testIndex :| [])) defaultIndexOptimizationSettings

1786 liftIO $ validateStatus resp 200

1799 liftIO $ validateStatus resp 200

1787

1800

1801+ describe "Suggest" $ do

1802+ it "returns a search suggestion using the phrase suggester" $ withTestEnv $ do

1803+ _ <- insertData

1804+ let {- query = QueryMatchNoneQuery

1805+ query = TermQuery (Term "user" "bitemyapp") Nothing -}

1806+ let phraseSuggester = mkPhraseSuggester (FieldName "message")

1807+ namedSuggester = Suggest "Use haskel" "suggest_name" (SuggestTypePhraseSuggester phraseSuggester)

1808+ search' = mkSearch Nothing Nothing

1809+ search = search' { suggestBody = Just namedSuggester }

1810+ expectedText = Just "use haskell"

1811+ resp <- searchByIndex testIndex search

1812+ parsed <- parseEsResponse resp :: BH IO (Either EsError (SearchResult Tweet))

1813+ case parsed of

1814+ Left e -> liftIO $ expectationFailure ("Expected an search suggestion but got " <> show e)

1815+ Right sr -> liftIO $ (suggestOptionsText . head . suggestResponseOptions . head . nsrResponses <$> suggest sr) `shouldBe` expectedText

1816+

1788 describe "JSON instances" $ do

1817 describe "JSON instances" $ do

1789 propJSON (Proxy :: Proxy Version)

1818 propJSON (Proxy :: Proxy Version)

1790 propJSON (Proxy :: Proxy IndexName)

1819 propJSON (Proxy :: Proxy IndexName)

@@ -1889,6 +1918,7 @@ main = hspec $ do

1889 propJSON (Proxy :: Proxy FSType)

1918 propJSON (Proxy :: Proxy FSType)

1890 propJSON (Proxy :: Proxy CompoundFormat)

1919 propJSON (Proxy :: Proxy CompoundFormat)

1891 propJSON (Proxy :: Proxy TemplateQueryInline)

1920 propJSON (Proxy :: Proxy TemplateQueryInline)

1921+ propJSON (Proxy :: Proxy Suggest)

1892

1922

1893-- Temporary solution for lacking of generic derivation of Arbitrary

1923-- Temporary solution for lacking of generic derivation of Arbitrary

1894-- We use generics-sop, as it's much more concise than directly using GHC.Generics

1924-- We use generics-sop, as it's much more concise than directly using GHC.Generics