4cc75a0 · typeable/bloodhound

git.iko.soy/typeable/bloodhound

on master help sign in

Auto id integration for the module reorg

4cc75a0Chris Allen9 years ago

bloodhound.cabal+3−1

compare with master

unifiedsplit

@@ -106,5 +106,7 @@ test-suite bloodhound-tests

106 exceptions,

107 temporary,

108 unix-compat,

109− network-uri

109+ network-uri,

110+ microlens,

111+ microlens-aeson

110 default-language: Haskell2010

112 default-language: Haskell2010

src/Database/V5/Bloodhound/Client.hs+18−1

@@ -858,7 +858,6 @@ deleteDocument (IndexName indexName)

858-- >>> _ <- runBH' $ refreshIndex testIndex

859bulk :: MonadBH m => V.Vector BulkOperation -> m Reply

860bulk bulkOps = do

861− liftIO $ print body

862 bindM2 post url (return body)

861 bindM2 post url (return body)

863 where url = joinPath ["_bulk"]

862 where url = joinPath ["_bulk"]

864 body = Just $ encodeBulkOperations bulkOps

863 body = Just $ encodeBulkOperations bulkOps

@@ -888,6 +887,12 @@ mkBulkStreamValue operation indexName mappingName docId =

888 , "_type" .= mappingName

887 , "_type" .= mappingName

889 , "_id" .= docId]]

888 , "_id" .= docId]]

890

889

890+mkBulkStreamValueAuto :: Text -> Text -> Text -> Value

891+mkBulkStreamValueAuto operation indexName mappingName =

892+ object [operation .=

893+ object [ "_index" .= indexName

894+ , "_type" .= mappingName]]

895+

891-- | 'encodeBulkOperation' is a convenience function for dumping a single 'BulkOperation'

896-- | 'encodeBulkOperation' is a convenience function for dumping a single 'BulkOperation'

892-- into an 'L.ByteString'

897-- into an 'L.ByteString'

893--

898--

@@ -901,6 +906,18 @@ encodeBulkOperation (BulkIndex (IndexName indexName)

901 where metadata = mkBulkStreamValue "index" indexName mappingName docId

906 where metadata = mkBulkStreamValue "index" indexName mappingName docId

902 blob = encode metadata `mappend` "\n" `mappend` encode value

907 blob = encode metadata `mappend` "\n" `mappend` encode value

903

908

909+encodeBulkOperation (BulkIndexAuto (IndexName indexName)

910+ (MappingName mappingName)

911+ value) = blob

912+ where metadata = mkBulkStreamValueAuto "index" indexName mappingName

913+ blob = encode metadata `mappend` "\n" `mappend` encode value

914+

915+encodeBulkOperation (BulkIndexEncodingAuto (IndexName indexName)

916+ (MappingName mappingName)

917+ encoding) = toLazyByteString blob

918+ where metadata = toEncoding (mkBulkStreamValueAuto "index" indexName mappingName)

919+ blob = fromEncoding metadata <> "\n" <> fromEncoding encoding

920+

904encodeBulkOperation (BulkCreate (IndexName indexName)

921encodeBulkOperation (BulkCreate (IndexName indexName)

905 (MappingName mappingName)

922 (MappingName mappingName)

906 (DocId docId) value) = blob

923 (DocId docId) value) = blob

src/Database/V5/Bloodhound/Types/Internal/Client.hs+20−3

@@ -358,15 +358,32 @@ data AllocationPolicy = AllocAll

358{-| 'BulkOperation' is a sum type for expressing the four kinds of bulk

359 operation index, create, delete, and update. 'BulkIndex' behaves like an

360 "upsert", 'BulkCreate' will fail if a document already exists at the DocId.

361−

362− <http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/docs-bulk.html#docs-bulk>

361+ Consult the <http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/docs-bulk.html#docs-bulk Bulk API documentation>

362+ for further explanation.

363+ Warning: Bulk operations suffixed with @Auto@ rely on ElasticSearch to

364+ generate the id. Often, people use auto-generated identifiers when

365+ ElasticSearch is the only place that their data is stored. Do not let

366+ ElasticSearch be the only place your data is stored. It does not guarantee

367+ durability, and it may silently discard data.

368+ This <https://github.com/elastic/elasticsearch/issues/10708 issue> is

369+ discussed further on github.

363-}

370-}

364data BulkOperation =

371data BulkOperation =

365 BulkIndex IndexName MappingName DocId Value

372 BulkIndex IndexName MappingName DocId Value

373+ -- ^ Create the document, replacing it if it already exists.

374+ | BulkIndexAuto IndexName MappingName Value

375+ -- ^ Create a document with an autogenerated id.

376+ | BulkIndexEncodingAuto IndexName MappingName Encoding

377+ -- ^ Create a document with an autogenerated id. Use fast JSON encoding.

366 | BulkCreate IndexName MappingName DocId Value

378 | BulkCreate IndexName MappingName DocId Value

379+ -- ^ Create a document, failing if it already exists.

367 | BulkCreateEncoding IndexName MappingName DocId Encoding

380 | BulkCreateEncoding IndexName MappingName DocId Encoding

381+ -- ^ Create a document, failing if it already exists. Use fast JSON encoding.

368 | BulkDelete IndexName MappingName DocId

382 | BulkDelete IndexName MappingName DocId

383+ -- ^ Delete the document

369− | BulkUpdate IndexName MappingName DocId Value deriving (Eq, Show)

384+ | BulkUpdate IndexName MappingName DocId Value

385+ -- ^ Update the document, merging the new value with the existing one.

386+ deriving (Eq, Show)

370

387

371{-| 'EsResult' describes the standard wrapper JSON document that you see in

388{-| 'EsResult' describes the standard wrapper JSON document that you see in

372 successful Elasticsearch lookups or lookups that couldn't find the document.

389 successful Elasticsearch lookups or lookups that couldn't find the document.

tests/V5/tests.hs+43−9

@@ -46,6 +46,9 @@ import Data.Typeable

46import qualified Data.Vector as V

47import qualified Data.Version as Vers

48import Database.V5.Bloodhound

49+import Lens.Micro

50+import Lens.Micro.Aeson

51+import qualified Lens.Micro.Aeson as LMA

49import Network.HTTP.Client hiding (Proxy)

52import Network.HTTP.Client hiding (Proxy)

50import qualified Network.HTTP.Types.Method as NHTM

53import qualified Network.HTTP.Types.Method as NHTM

51import qualified Network.HTTP.Types.Status as NHTS

54import qualified Network.HTTP.Types.Status as NHTS

@@ -59,7 +62,7 @@ import Test.QuickCheck.Property.Monoid (T (..), eq, prop_Monoid)

59import Test.Hspec.QuickCheck (prop)

62import Test.Hspec.QuickCheck (prop)

60import Test.QuickCheck

63import Test.QuickCheck

61import Test.QuickCheck.TH.Generators

64import Test.QuickCheck.TH.Generators

62−import Text.Pretty.Simple (pPrint)

65+-- import Text.Pretty.Simple (pPrint)

64testServer :: Server

67testServer :: Server

65testServer = Server "http://localhost:9200"

68testServer = Server "http://localhost:9200"

@@ -580,13 +583,13 @@ instance Arbitrary AliasRouting where

580 arbitrary = oneof [allAlias

583 arbitrary = oneof [allAlias

581 ,one

584 ,one

582 ,theOther

585 ,theOther

583− ,both]

586+ ,both']

584 where one = GranularAliasRouting

587 where one = GranularAliasRouting

585 <$> (Just <$> arbitrary)

588 <$> (Just <$> arbitrary)

586 <*> pure Nothing

589 <*> pure Nothing

587 theOther = GranularAliasRouting Nothing

590 theOther = GranularAliasRouting Nothing

588 <$> (Just <$> arbitrary)

591 <$> (Just <$> arbitrary)

589− both = GranularAliasRouting

592+ both' = GranularAliasRouting

590 <$> (Just <$> arbitrary)

593 <$> (Just <$> arbitrary)

591 <*> (Just <$> arbitrary)

594 <*> (Just <$> arbitrary)

592 allAlias = AllAliasRouting <$> arbitrary

595 allAlias = AllAliasRouting <$> arbitrary

@@ -1123,20 +1126,28 @@ main = hspec $ do

1123 let firstTest = BulkTest "blah"

1126 let firstTest = BulkTest "blah"

1124 let secondTest = BulkTest "bloo"

1127 let secondTest = BulkTest "bloo"

1125 let thirdTest = BulkTest "graffle"

1128 let thirdTest = BulkTest "graffle"

1129+ let fourthTest = BulkTest "garabadoo"

1130+ let fifthTest = BulkTest "serenity"

1126 let firstDoc = BulkIndex testIndex

1131 let firstDoc = BulkIndex testIndex

1127 testMapping (DocId "2") (toJSON firstTest)

1132 testMapping (DocId "2") (toJSON firstTest)

1128 let secondDoc = BulkCreate testIndex

1133 let secondDoc = BulkCreate testIndex

1129 testMapping (DocId "3") (toJSON secondTest)

1134 testMapping (DocId "3") (toJSON secondTest)

1130 let thirdDoc = BulkCreateEncoding testIndex

1135 let thirdDoc = BulkCreateEncoding testIndex

1131 testMapping (DocId "4") (toEncoding thirdTest)

1136 testMapping (DocId "4") (toEncoding thirdTest)

1137+ let fourthDoc = BulkIndexAuto testIndex

1138+ testMapping (toJSON fourthTest)

1139+ let fifthDoc = BulkIndexEncodingAuto testIndex

1140+ testMapping (toEncoding fifthTest)

1132− let stream = V.fromList [firstDoc, secondDoc, thirdDoc]

1141+ let stream = V.fromList [firstDoc, secondDoc, thirdDoc, fourthDoc, fifthDoc]

1133− bulkResp <- bulk stream

1142+ _ <- bulk stream

1134− liftIO $ pPrint bulkResp

1143+ -- liftIO $ pPrint bulkResp

1135− refreshResp <- refreshIndex testIndex

1144+ _ <- refreshIndex testIndex

1136− liftIO $ pPrint refreshResp

1145+ -- liftIO $ pPrint refreshResp

1137 fDoc <- getDocument testIndex testMapping (DocId "2")

1146 fDoc <- getDocument testIndex testMapping (DocId "2")

1138 sDoc <- getDocument testIndex testMapping (DocId "3")

1147 sDoc <- getDocument testIndex testMapping (DocId "3")

1139 tDoc <- getDocument testIndex testMapping (DocId "4")

1148 tDoc <- getDocument testIndex testMapping (DocId "4")

1149+ -- note that we cannot query for fourthDoc and fifthDoc since we

1150+ -- do not know their autogenerated ids.

1140 let maybeFirst =

1151 let maybeFirst =

1141 eitherDecode

1152 eitherDecode

1142 $ responseBody fDoc

1153 $ responseBody fDoc

@@ -1149,11 +1160,34 @@ main = hspec $ do

1149 eitherDecode

1160 eitherDecode

1150 $ responseBody tDoc

1161 $ responseBody tDoc

1151 :: Either String (EsResult BulkTest)

1162 :: Either String (EsResult BulkTest)

1152− liftIO $ pPrint [maybeFirst, maybeSecond, maybeThird]

1163+ -- liftIO $ pPrint [maybeFirst, maybeSecond, maybeThird]

1153 liftIO $ do

1164 liftIO $ do

1154 fmap getSource maybeFirst `shouldBe` Right (Just firstTest)

1165 fmap getSource maybeFirst `shouldBe` Right (Just firstTest)

1155 fmap getSource maybeSecond `shouldBe` Right (Just secondTest)

1166 fmap getSource maybeSecond `shouldBe` Right (Just secondTest)

1156 fmap getSource maybeThird `shouldBe` Right (Just thirdTest)

1167 fmap getSource maybeThird `shouldBe` Right (Just thirdTest)

1168+ -- Since we can't get the docs by doc id, we check for their existence in

1169+ -- a match all query.

1170+ let query = MatchAllQuery Nothing

1171+ let search = mkSearch (Just query) Nothing

1172+ resp <- searchByIndex testIndex search

1173+ parsed <- parseEsResponse resp :: BH IO (Either EsError (SearchResult Value))

1174+ case parsed of

1175+ Left e ->

1176+ liftIO $ expectationFailure ("Expected a script-transformed result but got: " <> show e)

1177+ (Right sr) -> do

1178+ liftIO $

1179+ hitsTotal (searchHits sr) `shouldBe` 6

1180+ let nameList :: [Text]

1181+ nameList =

1182+ (hits (searchHits sr))

1183+ ^.. traverse

1184+ . to hitSource

1185+ . _Just

1186+ . LMA.key "name"

1187+ . _String

1188+ liftIO $

1189+ nameList

1190+ `shouldBe` ["blah","bloo","graffle","garabadoo","serenity"]

1157

1191

1158

1192

1159 describe "query API" $ do

1193 describe "query API" $ do