984ca41 · typeable/bloodhound

git.iko.soy/typeable/bloodhound

Merge remote-tracking branch 'upstream/master' into boost-in-match-query

Conflicts: src/Database/Bloodhound/Client.hs

commit984ca41e776b30e7b3089b77fcb84ffe62362222 authorThomas van Noort <thomas@silk.co> dateMon 31 Aug 2015, 10:15 (11 years ago) parente51b515 68ff8c2

3 files changed+89−13

compare with master

unifiedsplit

src/Database/Bloodhound/Client.hs+6−6 src/Database/Bloodhound/Types.hs+46−6 tests/tests.hs+37−1

src/Database/Bloodhound/Client.hs+6−6

@@ -586,9 +586,9 @@ scanSearch indexName mappingName search = do

586586--

587587-- >>> let query = TermQuery (Term "user" "bitemyapp") Nothing

588588-- >>> mkSearch (Just query) Nothing

589−-- Search {queryBody = Just (TermQuery (Term {termField = "user", termValue = "bitemyapp"}) Nothing), filterBody = Nothing, sortBody = Nothing, aggBody = Nothing, highlight = Nothing, trackSortScores = False, from = From 0, size = Size 10, searchType = SearchTypeQueryThenFetch, fields = Nothing}

589+-- Search {queryBody = Just (TermQuery (Term {termField = "user", termValue = "bitemyapp"}) Nothing), filterBody = Nothing, sortBody = Nothing, aggBody = Nothing, highlight = Nothing, trackSortScores = False, from = From 0, size = Size 10, searchType = SearchTypeQueryThenFetch, fields = Nothing, source = Nothing}

590590mkSearch :: Maybe Query -> Maybe Filter -> Search

591−mkSearch query filter = Search query filter Nothing Nothing Nothing False (From 0) (Size 10) SearchTypeQueryThenFetch Nothing

591+mkSearch query filter = Search query filter Nothing Nothing Nothing False (From 0) (Size 10) SearchTypeQueryThenFetch Nothing Nothing

592592

593593-- | 'mkAggregateSearch' is a helper function that defaults everything in a 'Search' except for

594594-- the 'Query' and the 'Aggregation'.

@@ -598,7 +598,7 @@ mkSearch query filter = Search query filter Nothing Nothing Nothing False (From

598598-- TermsAgg (TermsAggregation {term = Left "user", termInclude = Nothing, termExclude = Nothing, termOrder = Nothing, termMinDocCount = Nothing, termSize = Nothing, termShardSize = Nothing, termCollectMode = Just BreadthFirst, termExecutionHint = Nothing, termAggs = Nothing})

599599-- >>> let myAggregation = mkAggregateSearch Nothing $ mkAggregations "users" terms

600600mkAggregateSearch :: Maybe Query -> Aggregations -> Search

601−mkAggregateSearch query mkSearchAggs = Search query Nothing Nothing (Just mkSearchAggs) Nothing False (From 0) (Size 0) SearchTypeQueryThenFetch Nothing

601+mkAggregateSearch query mkSearchAggs = Search query Nothing Nothing (Just mkSearchAggs) Nothing False (From 0) (Size 0) SearchTypeQueryThenFetch Nothing Nothing

602602

603603-- | 'mkHighlightSearch' is a helper function that defaults everything in a 'Search' except for

604604-- the 'Query' and the 'Aggregation'.

@@ -607,7 +607,7 @@ mkAggregateSearch query mkSearchAggs = Search query Nothing Nothing (Just mkSear

607607-- >>> let testHighlight = Highlights Nothing [FieldHighlight (FieldName "message") Nothing]

608608-- >>> let search = mkHighlightSearch (Just query) testHighlight

609609mkHighlightSearch :: Maybe Query -> Highlights -> Search

610−mkHighlightSearch query searchHighlights = Search query Nothing Nothing Nothing (Just searchHighlights) False (From 0) (Size 10) SearchTypeQueryThenFetch Nothing

610+mkHighlightSearch query searchHighlights = Search query Nothing Nothing Nothing (Just searchHighlights) False (From 0) (Size 10) SearchTypeQueryThenFetch Nothing Nothing

611611

612612-- | 'pageSearch' is a helper function that takes a search and assigns the from

613613-- and size fields for the search. The from parameter defines the offset

@@ -617,9 +617,9 @@ mkHighlightSearch query searchHighlights = Search query Nothing Nothing Nothing

617617-- >>> let query = QueryMatchQuery $ mkMatchQuery (FieldName "_all") (QueryString "haskell")

618618-- >>> let search = mkSearch (Just query) Nothing

619619-- >>> search

620−-- Search {queryBody = Just (QueryMatchQuery (MatchQuery {matchQueryField = FieldName "_all", matchQueryQueryString = QueryString "haskell", matchQueryOperator = Or, matchQueryZeroTerms = ZeroTermsNone, matchQueryCutoffFrequency = Nothing, matchQueryMatchType = Nothing, matchQueryAnalyzer = Nothing, matchQueryMaxExpansions = Nothing, matchQueryLenient = Nothing, matchQueryBoost = Nothing})), filterBody = Nothing, sortBody = Nothing, aggBody = Nothing, highlight = Nothing, trackSortScores = False, from = From 0, size = Size 10, searchType = SearchTypeQueryThenFetch, fields = Nothing}

620+-- Search {queryBody = Just (QueryMatchQuery (MatchQuery {matchQueryField = FieldName "_all", matchQueryQueryString = QueryString "haskell", matchQueryOperator = Or, matchQueryZeroTerms = ZeroTermsNone, matchQueryCutoffFrequency = Nothing, matchQueryMatchType = Nothing, matchQueryAnalyzer = Nothing, matchQueryMaxExpansions = Nothing, matchQueryLenient = Nothing, matchQueryBoost = Nothing})), filterBody = Nothing, sortBody = Nothing, aggBody = Nothing, highlight = Nothing, trackSortScores = False, from = From 0, size = Size 10, searchType = SearchTypeQueryThenFetch, fields = Nothing, source = Nothing}

621621-- >>> pageSearch (From 10) (Size 100) search

622−-- Search {queryBody = Just (QueryMatchQuery (MatchQuery {matchQueryField = FieldName "_all", matchQueryQueryString = QueryString "haskell", matchQueryOperator = Or, matchQueryZeroTerms = ZeroTermsNone, matchQueryCutoffFrequency = Nothing, matchQueryMatchType = Nothing, matchQueryAnalyzer = Nothing, matchQueryMaxExpansions = Nothing, matchQueryLenient = Nothing, matchQueryBoost = Nothing})), filterBody = Nothing, sortBody = Nothing, aggBody = Nothing, highlight = Nothing, trackSortScores = False, from = From 10, size = Size 100, searchType = SearchTypeQueryThenFetch, fields = Nothing}

622+-- Search {queryBody = Just (QueryMatchQuery (MatchQuery {matchQueryField = FieldName "_all", matchQueryQueryString = QueryString "haskell", matchQueryOperator = Or, matchQueryZeroTerms = ZeroTermsNone, matchQueryCutoffFrequency = Nothing, matchQueryMatchType = Nothing, matchQueryAnalyzer = Nothing, matchQueryMaxExpansions = Nothing, matchQueryLenient = Nothing, matchQueryBoost = Nothing})), filterBody = Nothing, sortBody = Nothing, aggBody = Nothing, highlight = Nothing, trackSortScores = False, from = From 10, size = Size 100, searchType = SearchTypeQueryThenFetch, fields = Nothing, source = Nothing}

623623pageSearch :: From -- ^ The result offset

624624 -> Size -- ^ The number of results to return

625625 -> Search -- ^ The current seach

src/Database/Bloodhound/Types.hs+46−6

@@ -75,6 +75,11 @@ module Database.Bloodhound.Types

7575 , TrackSortScores

7676 , From(..)

7777 , Size(..)

78+ , Source(..)

79+ , PatternOrPatterns(..)

80+ , Include(..)

81+ , Exclude(..)

82+ , Pattern(..)

7883 , ShardResult(..)

7984 , Hit(..)

8085 , Filter(..)

@@ -733,11 +738,12 @@ data Search = Search { queryBody :: Maybe Query

733738 -- default False

734739 , trackSortScores :: TrackSortScores

735740 , from :: From

736− , size :: Size

741+ , size :: Size

737742 , searchType :: SearchType

738− , fields :: Maybe [FieldName] } deriving (Eq, Show)

743+ , fields :: Maybe [FieldName]

744+ , source :: Maybe Source } deriving (Eq, Show)

739745

740−data SearchType = SearchTypeQueryThenFetch

746+data SearchType = SearchTypeQueryThenFetch

741747 | SearchTypeDfsQueryThenFetch

742748 | SearchTypeCount

743749 | SearchTypeScan

@@ -745,6 +751,20 @@ data SearchType = SearchTypeQueryThenFetch

745751 | SearchTypeDfsQueryAndFetch

746752 deriving (Eq, Show)

747753

754+data Source =

755+ NoSource

756+ | SourcePatterns PatternOrPatterns

757+ | SourceIncludeExclude Include Exclude

758+ deriving (Show, Eq)

759+

760+data PatternOrPatterns = PopPattern Pattern

761+ | PopPatterns [Pattern] deriving (Eq, Show)

762+

763+data Include = Include [Pattern] deriving (Eq, Show)

764+data Exclude = Exclude [Pattern] deriving (Eq, Show)

765+

766+newtype Pattern = Pattern Text deriving (Eq, Show)

767+

748768data Highlights = Highlights { globalsettings :: Maybe HighlightSettings

749769 , highlightFields :: [FieldHighlight]

750770 } deriving (Show, Eq)

@@ -1257,7 +1277,7 @@ data SearchResult a =

12571277 , timedOut :: Bool

12581278 , shards :: ShardResult

12591279 , searchHits :: SearchHits a

1260− , aggregations :: Maybe AggregationResults

1280+ , aggregations :: Maybe AggregationResults

12611281 , scrollId :: Maybe ScrollId } deriving (Eq, Show)

12621282

12631283type ScrollId = Text -- Fixme: Newtype

@@ -2110,7 +2130,7 @@ instance (FromJSON a) => FromJSON (EsResultFound a) where

21102130 parseJSON _ = empty

21112131

21122132instance ToJSON Search where

2113− toJSON (Search query sFilter sort searchAggs highlight sTrackSortScores sFrom sSize _ sFields) =

2133+ toJSON (Search query sFilter sort searchAggs highlight sTrackSortScores sFrom sSize _ sFields sSource) =

21142134 omitNulls [ "query" .= query

21152135 , "filter" .= sFilter

21162136 , "sort" .= sort

@@ -2119,7 +2139,27 @@ instance ToJSON Search where

21192139 , "from" .= sFrom

21202140 , "size" .= sSize

21212141 , "track_scores" .= sTrackSortScores

2122− , "fields" .= sFields]

2142+ , "fields" .= sFields

2143+ , "_source" .= sSource]

2144+

2145+

2146+instance ToJSON Source where

2147+ toJSON NoSource = toJSON False

2148+ toJSON (SourcePatterns patterns) = toJSON patterns

2149+ toJSON (SourceIncludeExclude incl excl) = object [ "include" .= incl, "exclude" .= excl ]

2150+

2151+instance ToJSON PatternOrPatterns where

2152+ toJSON (PopPattern pattern) = toJSON pattern

2153+ toJSON (PopPatterns patterns) = toJSON patterns

2154+

2155+instance ToJSON Include where

2156+ toJSON (Include patterns) = toJSON patterns

2157+

2158+instance ToJSON Exclude where

2159+ toJSON (Exclude patterns) = toJSON patterns

2160+

2161+instance ToJSON Pattern where

2162+ toJSON (Pattern pattern) = toJSON pattern

21232163

21242164

21252165instance ToJSON FieldHighlight where

tests/tests.hs+37−1

@@ -220,6 +220,17 @@ searchTweetHighlight search = do

220220 let myHighlight = fmap (hitHighlight . head . hits . searchHits) result

221221 return myHighlight

222222

223+searchExpectSource :: Source -> Either String Value -> BH IO ()

224+searchExpectSource src expected = do

225+ _ <- insertData

226+ let query = QueryMatchQuery $ mkMatchQuery (FieldName "_all") (QueryString "haskell")

227+ let search = (mkSearch (Just query) Nothing) { source = Just src }

228+ reply <- searchAll search

229+ let result = eitherDecode (responseBody reply) :: Either String (SearchResult Value)

230+ let value = fmap (hitSource . head . hits . searchHits) result

231+ liftIO $

232+ value `shouldBe` expected

233+

223234data BulkTest = BulkTest { name :: Text } deriving (Eq, Generic, Show)

224235instance FromJSON BulkTest

225236instance ToJSON BulkTest

@@ -435,7 +446,7 @@ main = hspec $ do

435446 let sortSpec = DefaultSortSpec $ mkSort (FieldName "age") Ascending

436447 let search = Search Nothing

437448 (Just IdentityFilter) (Just [sortSpec]) Nothing Nothing

438− False (From 0) (Size 10) SearchTypeQueryThenFetch Nothing

449+ False (From 0) (Size 10) SearchTypeQueryThenFetch Nothing Nothing

439450 result <- searchTweets search

440451 let myTweet = fmap (hitSource . head . hits . searchHits) result

441452 liftIO $

@@ -698,7 +709,32 @@ main = hspec $ do

698709 liftIO $

699710 myHighlight `shouldBe` Right Nothing

700711

712+ describe "Source filtering" $ do

713+

714+ it "doesn't include source when sources are disabled" $ withTestEnv $ do

715+ searchExpectSource

716+ NoSource

717+ (Left "key \"_source\" not present")

718+

719+ it "includes a source" $ withTestEnv $ do

720+ searchExpectSource

721+ (SourcePatterns (PopPattern (Pattern "message")))

722+ (Right (Object (HM.fromList [("message", String "Use haskell!")])))

723+

724+ it "includes sources" $ withTestEnv $ do

725+ searchExpectSource

726+ (SourcePatterns (PopPatterns [Pattern "user", Pattern "message"]))

727+ (Right (Object (HM.fromList [("user",String "bitemyapp"),("message", String "Use haskell!")])))

728+

729+ it "includes source patterns" $ withTestEnv $ do

730+ searchExpectSource

731+ (SourcePatterns (PopPattern (Pattern "*ge")))

732+ (Right (Object (HM.fromList [("age", Number 10000),("message", String "Use haskell!")])))

701733

734+ it "excludes source patterns" $ withTestEnv $ do

735+ searchExpectSource

736+ (SourceIncludeExclude (Include []) (Exclude [Pattern "l*", Pattern "*ge", Pattern "postDate"]))

737+ (Right (Object (HM.fromList [("user",String "bitemyapp")])))

702738

703739 describe "ToJSON RegexpFlags" $ do

704740 it "generates the correct JSON for AllRegexpFlags" $