4cc75a0 · typeable/bloodhound

git.iko.soy/typeable/bloodhound

Auto id integration for the module reorg

commit4cc75a0d983fc247835da6ea86c30ece0a77dcf9 authorChris Allen <cma@bitemyapp.com> dateThu 8 Feb 2018, 17:29 (9 years ago) parentf7d3149

4 files changed+84−14

compare with master

unifiedsplit

bloodhound.cabal+3−1 src/Database/V5/Bloodhound/Client.hs+18−1 src/Database/V5/Bloodhound/Types/Internal/Client.hs+20−3 tests/V5/tests.hs+43−9

bloodhound.cabal+3−1

@@ -106,5 +106,7 @@ test-suite bloodhound-tests

106106 exceptions,

107107 temporary,

108108 unix-compat,

109− network-uri

109+ network-uri,

110+ microlens,

111+ microlens-aeson

110112 default-language: Haskell2010

src/Database/V5/Bloodhound/Client.hs+18−1

@@ -858,7 +858,6 @@ deleteDocument (IndexName indexName)

858858-- >>> _ <- runBH' $ refreshIndex testIndex

859859bulk :: MonadBH m => V.Vector BulkOperation -> m Reply

860860bulk bulkOps = do

861− liftIO $ print body

862861 bindM2 post url (return body)

863862 where url = joinPath ["_bulk"]

864863 body = Just $ encodeBulkOperations bulkOps

@@ -888,6 +887,12 @@ mkBulkStreamValue operation indexName mappingName docId =

888887 , "_type" .= mappingName

889888 , "_id" .= docId]]

890889

890+mkBulkStreamValueAuto :: Text -> Text -> Text -> Value

891+mkBulkStreamValueAuto operation indexName mappingName =

892+ object [operation .=

893+ object [ "_index" .= indexName

894+ , "_type" .= mappingName]]

895+

891896-- | 'encodeBulkOperation' is a convenience function for dumping a single 'BulkOperation'

892897-- into an 'L.ByteString'

893898--

@@ -901,6 +906,18 @@ encodeBulkOperation (BulkIndex (IndexName indexName)

901906 where metadata = mkBulkStreamValue "index" indexName mappingName docId

902907 blob = encode metadata `mappend` "\n" `mappend` encode value

903908

909+encodeBulkOperation (BulkIndexAuto (IndexName indexName)

910+ (MappingName mappingName)

911+ value) = blob

912+ where metadata = mkBulkStreamValueAuto "index" indexName mappingName

913+ blob = encode metadata `mappend` "\n" `mappend` encode value

914+

915+encodeBulkOperation (BulkIndexEncodingAuto (IndexName indexName)

916+ (MappingName mappingName)

917+ encoding) = toLazyByteString blob

918+ where metadata = toEncoding (mkBulkStreamValueAuto "index" indexName mappingName)

919+ blob = fromEncoding metadata <> "\n" <> fromEncoding encoding

920+

904921encodeBulkOperation (BulkCreate (IndexName indexName)

905922 (MappingName mappingName)

906923 (DocId docId) value) = blob

src/Database/V5/Bloodhound/Types/Internal/Client.hs+20−3

@@ -358,15 +358,32 @@ data AllocationPolicy = AllocAll

358358{-| 'BulkOperation' is a sum type for expressing the four kinds of bulk

359359 operation index, create, delete, and update. 'BulkIndex' behaves like an

360360 "upsert", 'BulkCreate' will fail if a document already exists at the DocId.

361−

362− <http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/docs-bulk.html#docs-bulk>

361+ Consult the <http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/docs-bulk.html#docs-bulk Bulk API documentation>

362+ for further explanation.

363+ Warning: Bulk operations suffixed with @Auto@ rely on ElasticSearch to

364+ generate the id. Often, people use auto-generated identifiers when

365+ ElasticSearch is the only place that their data is stored. Do not let

366+ ElasticSearch be the only place your data is stored. It does not guarantee

367+ durability, and it may silently discard data.

368+ This <https://github.com/elastic/elasticsearch/issues/10708 issue> is

369+ discussed further on github.

363370-}

364371data BulkOperation =

365372 BulkIndex IndexName MappingName DocId Value

373+ -- ^ Create the document, replacing it if it already exists.

374+ | BulkIndexAuto IndexName MappingName Value

375+ -- ^ Create a document with an autogenerated id.

376+ | BulkIndexEncodingAuto IndexName MappingName Encoding

377+ -- ^ Create a document with an autogenerated id. Use fast JSON encoding.

366378 | BulkCreate IndexName MappingName DocId Value

379+ -- ^ Create a document, failing if it already exists.

367380 | BulkCreateEncoding IndexName MappingName DocId Encoding

381+ -- ^ Create a document, failing if it already exists. Use fast JSON encoding.

368382 | BulkDelete IndexName MappingName DocId

369− | BulkUpdate IndexName MappingName DocId Value deriving (Eq, Show)

383+ -- ^ Delete the document

384+ | BulkUpdate IndexName MappingName DocId Value

385+ -- ^ Update the document, merging the new value with the existing one.

386+ deriving (Eq, Show)

370387

371388{-| 'EsResult' describes the standard wrapper JSON document that you see in

372389 successful Elasticsearch lookups or lookups that couldn't find the document.

tests/V5/tests.hs+43−9

@@ -46,6 +46,9 @@ import Data.Typeable

4646import qualified Data.Vector as V

4747import qualified Data.Version as Vers

4848import Database.V5.Bloodhound

49+import Lens.Micro

50+import Lens.Micro.Aeson

51+import qualified Lens.Micro.Aeson as LMA

4952import Network.HTTP.Client hiding (Proxy)

5053import qualified Network.HTTP.Types.Method as NHTM

5154import qualified Network.HTTP.Types.Status as NHTS

@@ -59,7 +62,7 @@ import Test.QuickCheck.Property.Monoid (T (..), eq, prop_Monoid)

5962import Test.Hspec.QuickCheck (prop)

6063import Test.QuickCheck

6164import Test.QuickCheck.TH.Generators

62−import Text.Pretty.Simple (pPrint)

65+-- import Text.Pretty.Simple (pPrint)

6366

6467testServer :: Server

6568testServer = Server "http://localhost:9200"

@@ -580,13 +583,13 @@ instance Arbitrary AliasRouting where

580583 arbitrary = oneof [allAlias

581584 ,one

582585 ,theOther

583− ,both]

586+ ,both']

584587 where one = GranularAliasRouting

585588 <$> (Just <$> arbitrary)

586589 <*> pure Nothing

587590 theOther = GranularAliasRouting Nothing

588591 <$> (Just <$> arbitrary)

589− both = GranularAliasRouting

592+ both' = GranularAliasRouting

590593 <$> (Just <$> arbitrary)

591594 <*> (Just <$> arbitrary)

592595 allAlias = AllAliasRouting <$> arbitrary

@@ -1123,20 +1126,28 @@ main = hspec $ do

11231126 let firstTest = BulkTest "blah"

11241127 let secondTest = BulkTest "bloo"

11251128 let thirdTest = BulkTest "graffle"

1129+ let fourthTest = BulkTest "garabadoo"

1130+ let fifthTest = BulkTest "serenity"

11261131 let firstDoc = BulkIndex testIndex

11271132 testMapping (DocId "2") (toJSON firstTest)

11281133 let secondDoc = BulkCreate testIndex

11291134 testMapping (DocId "3") (toJSON secondTest)

11301135 let thirdDoc = BulkCreateEncoding testIndex

11311136 testMapping (DocId "4") (toEncoding thirdTest)

1132− let stream = V.fromList [firstDoc, secondDoc, thirdDoc]

1133− bulkResp <- bulk stream

1134− liftIO $ pPrint bulkResp

1135− refreshResp <- refreshIndex testIndex

1136− liftIO $ pPrint refreshResp

1137+ let fourthDoc = BulkIndexAuto testIndex

1138+ testMapping (toJSON fourthTest)

1139+ let fifthDoc = BulkIndexEncodingAuto testIndex

1140+ testMapping (toEncoding fifthTest)

1141+ let stream = V.fromList [firstDoc, secondDoc, thirdDoc, fourthDoc, fifthDoc]

1142+ _ <- bulk stream

1143+ -- liftIO $ pPrint bulkResp

1144+ _ <- refreshIndex testIndex

1145+ -- liftIO $ pPrint refreshResp

11371146 fDoc <- getDocument testIndex testMapping (DocId "2")

11381147 sDoc <- getDocument testIndex testMapping (DocId "3")

11391148 tDoc <- getDocument testIndex testMapping (DocId "4")

1149+ -- note that we cannot query for fourthDoc and fifthDoc since we

1150+ -- do not know their autogenerated ids.

11401151 let maybeFirst =

11411152 eitherDecode

11421153 $ responseBody fDoc

@@ -1149,11 +1160,34 @@ main = hspec $ do

11491160 eitherDecode

11501161 $ responseBody tDoc

11511162 :: Either String (EsResult BulkTest)

1152− liftIO $ pPrint [maybeFirst, maybeSecond, maybeThird]

1163+ -- liftIO $ pPrint [maybeFirst, maybeSecond, maybeThird]

11531164 liftIO $ do

11541165 fmap getSource maybeFirst `shouldBe` Right (Just firstTest)

11551166 fmap getSource maybeSecond `shouldBe` Right (Just secondTest)

11561167 fmap getSource maybeThird `shouldBe` Right (Just thirdTest)

1168+ -- Since we can't get the docs by doc id, we check for their existence in

1169+ -- a match all query.

1170+ let query = MatchAllQuery Nothing

1171+ let search = mkSearch (Just query) Nothing

1172+ resp <- searchByIndex testIndex search

1173+ parsed <- parseEsResponse resp :: BH IO (Either EsError (SearchResult Value))

1174+ case parsed of

1175+ Left e ->

1176+ liftIO $ expectationFailure ("Expected a script-transformed result but got: " <> show e)

1177+ (Right sr) -> do

1178+ liftIO $

1179+ hitsTotal (searchHits sr) `shouldBe` 6

1180+ let nameList :: [Text]

1181+ nameList =

1182+ (hits (searchHits sr))

1183+ ^.. traverse

1184+ . to hitSource

1185+ . _Just

1186+ . LMA.key "name"

1187+ . _String

1188+ liftIO $

1189+ nameList

1190+ `shouldBe` ["blah","bloo","graffle","garabadoo","serenity"]

11571191

11581192

11591193 describe "query API" $ do