From a4a15751a9e094b8e32a670c3562d3bdf0e28a0d Mon Sep 17 00:00:00 2001 From: Josh Berman Date: Tue, 18 Aug 2015 16:26:51 +0300 Subject: [PATCH 1/2] url query encoding bug --- bloodhound.cabal | 2 +- src/Database/Bloodhound/Client.hs | 18 +++++++++--------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/bloodhound.cabal b/bloodhound.cabal index 0c7a983..ee634b8 100644 --- a/bloodhound.cabal +++ b/bloodhound.cabal @@ -1,5 +1,5 @@ name: bloodhound -version: 0.7.1.0 +version: 0.7.1.1 synopsis: ElasticSearch client library for Haskell description: ElasticSearch made awesome for Haskell hackers homepage: https://github.com/bitemyapp/bloodhound diff --git a/src/Database/Bloodhound/Client.hs b/src/Database/Bloodhound/Client.hs index 2d6be13..e369a9a 100644 --- a/src/Database/Bloodhound/Client.hs +++ b/src/Database/Bloodhound/Client.hs @@ -173,16 +173,16 @@ joinPath ps = do return $ joinPath' (s:ps) appendSearchTypeParam :: Text -> SearchType -> Text -appendSearchTypeParam originalUrl st = addQuery [(keyEq, Just stParams)] originalUrl - where keyEq = "search_type=" - stParams - | st == SearchTypeDfsQueryThenFetch = "dfs_query_then_fetch" - | st == SearchTypeCount = "count" - | st == SearchTypeScan = "scan&scroll=1m" - | st == SearchTypeQueryAndFetch = "query_and_fetch" - | st == SearchTypeDfsQueryAndFetch = "dfs_query_and_fetch" +appendSearchTypeParam originalUrl st = addQuery params originalUrl + where stText = "search_type" + params + | st == SearchTypeDfsQueryThenFetch = [(stText, Just "dfs_query_then_fetch")] + | st == SearchTypeCount = [(stText, Just "count")] + | st == SearchTypeScan = [(stText, Just "scan"), ("scroll", Just "1m")] + | st == SearchTypeQueryAndFetch = [(stText, Just "query_and_fetch")] + | st == SearchTypeDfsQueryAndFetch = [(stText, Just "dfs_query_and_fetch")] -- used to catch 'SearchTypeQueryThenFetch', which is also the default - | otherwise = "query_then_fetch" + | otherwise = [(stText, Just "query_then_fetch")] -- | Severely dumbed down query renderer. Assumes your data doesn't -- need any encoding From fa9cff1ce8c4a699744159c196d618274ade99ff Mon Sep 17 00:00:00 2001 From: Josh Berman Date: Sun, 23 Aug 2015 16:05:35 +0300 Subject: [PATCH 2/2] limit change scanSearch to given index and mapping to better match existing search functionality; add tests --- bloodhound.cabal | 2 +- src/Database/Bloodhound/Client.hs | 14 ++++++++------ tests/tests.hs | 13 +++++++++++++ 3 files changed, 22 insertions(+), 7 deletions(-) diff --git a/bloodhound.cabal b/bloodhound.cabal index ee634b8..795b5ef 100644 --- a/bloodhound.cabal +++ b/bloodhound.cabal @@ -1,5 +1,5 @@ name: bloodhound -version: 0.7.1.1 +version: 0.7.2.0 synopsis: ElasticSearch client library for Haskell description: ElasticSearch made awesome for Haskell hackers homepage: https://github.com/bitemyapp/bloodhound diff --git a/src/Database/Bloodhound/Client.hs b/src/Database/Bloodhound/Client.hs index e369a9a..3de93f8 100644 --- a/src/Database/Bloodhound/Client.hs +++ b/src/Database/Bloodhound/Client.hs @@ -543,9 +543,9 @@ searchByType (IndexName indexName) (MappingName mappingName) = bindM2 dispatchSearch url . return where url = joinPath [indexName, mappingName, "_search"] -scanSearch' :: MonadBH m => Search -> m (Maybe ScrollId) -scanSearch' search = do - let url = joinPath ["_search"] +scanSearch' :: MonadBH m => IndexName -> MappingName -> Search -> m (Maybe ScrollId) +scanSearch' (IndexName indexName) (MappingName mappingName) search = do + let url = joinPath [indexName, mappingName, "_search"] search' = search { searchType = SearchTypeScan } resp' <- bindM2 dispatchSearch url (return search') let msr = decode' $ responseBody resp' :: Maybe (SearchResult ()) @@ -570,9 +570,11 @@ simpleAccumilator oldHits (newHits, msid) = do (newHits', msid') <- scroll' msid simpleAccumilator (oldHits ++ newHits) (newHits', msid') -scanSearch :: (FromJSON a, MonadBH m) => Search -> m [Hit a] -scanSearch search = do - msid <- scanSearch' search +-- | 'scanSearch' uses the 'scan&scroll' API of elastic, +-- for a given 'IndexName' and 'MappingName', +scanSearch :: (FromJSON a, MonadBH m) => IndexName -> MappingName -> Search -> m [Hit a] +scanSearch indexName mappingName search = do + msid <- scanSearch' indexName mappingName search (hits, msid') <- scroll' msid (totalHits, _) <- simpleAccumilator [] (hits, msid') return totalHits diff --git a/tests/tests.hs b/tests/tests.hs index e4206f5..de8ba4d 100644 --- a/tests/tests.hs +++ b/tests/tests.hs @@ -762,3 +762,16 @@ main = hspec $ do enumFrom (pred maxBound :: DocVersion) `shouldBe` [pred maxBound, maxBound] enumFrom (pred maxBound :: DocVersion) `shouldBe` [pred maxBound, maxBound] enumFromThen minBound (pred maxBound :: DocVersion) `shouldBe` [minBound, pred maxBound] + + describe "scan&scroll API" $ do + it "returns documents using the scan&scroll API" $ withTestEnv $ do + _ <- insertData + _ <- insertOther + let search = (mkSearch (Just $ MatchAllQuery Nothing) Nothing) { size = (Size 1) } + regular_search <- searchTweet search + scan_search' <- scanSearch testIndex testMapping search :: BH IO [Hit Tweet] + let scan_search = map hitSource scan_search' + liftIO $ + regular_search `shouldBe` Right exampleTweet -- Check that the size restrtiction is being honored + liftIO $ + scan_search `shouldMatchList` [exampleTweet, otherTweet]