git.iko.soy/typeable/bloodhound
on master help sign in
Merge pull request #45 from Soostone/44-versioning

44 versioning

commit404d807ee3e4a547f608ae16f91aae01018ef71c authorChris Allen <cma@bitemyapp.com> dateFri 10 Jul 2015, 16:38 (11 years ago) parentc7e041242500b1
5 files changed+194−14
README.md+2−2 bloodhound.cabal+2−1 src/Database/Bloodhound/Client.hs+46−4 src/Database/Bloodhound/Types.hs+101−1 tests/tests.hs+43−6
README.md+2−2
@@ -225,7 +225,7 @@ instance FromJSON Location
225225-- λ> encode $ Location 10.0 10.0
226226-- "{\"lat\":10,\"lon\":10}"
227227
228−resp <- withBH' $ indexDocument testIndex testMapping exampleTweet (DocId "1")
228+resp <- withBH' $ indexDocument testIndex testMapping defaultIndexDocumentSettings exampleTweet (DocId "1")
229229
230230```
231231
@@ -368,7 +368,7 @@ let encodedOperations = encodeBulkOperations stream
368368
369369-- to insert into a particular server
370370-- bulk :: V.Vector BulkOperation -> IO Reply
371−_ <- withBH' $ bulk stream
371+_ <- withBH' $ bulk streamp
372372
373373```
374374
bloodhound.cabal+2−1
@@ -37,7 +37,8 @@ library
3737 vector >= 0.10.9 && <0.11,
3838 uri-bytestring >= 0.1 && <0.2,
3939 exceptions,
40− data-default-class
40+ data-default-class,
41+ blaze-builder
4142 default-language: Haskell2010
4243
4344test-suite tests
src/Database/Bloodhound/Client.hs+46−4
@@ -47,9 +47,14 @@ module Database.Bloodhound.Client
4747 , getStatus
4848 , encodeBulkOperations
4949 , encodeBulkOperation
50+ -- * Reply-handling tools
51+ , isVersionConflict
52+ , isSuccess
53+ , isCreated
5054 )
5155 where
5256
57+import qualified Blaze.ByteString.Builder as BB
5358import Control.Applicative
5459import Control.Monad
5560import Control.Monad.Catch
@@ -58,7 +63,9 @@ import Data.Aeson
5863import Data.ByteString.Lazy.Builder
5964import qualified Data.ByteString.Lazy.Char8 as L
6065import Data.Default.Class
66+import Data.Ix
6167import Data.Maybe (fromMaybe)
68+import Data.Monoid
6269import Data.Text (Text)
6370import qualified Data.Text as T
6471import qualified Data.Text.Encoding as T
@@ -66,6 +73,7 @@ import qualified Data.Vector as V
6673import Network.HTTP.Client
6774import qualified Network.HTTP.Types.Method as NHTM
6875import qualified Network.HTTP.Types.Status as NHTS
76+import qualified Network.HTTP.Types.URI as NHTU
6977import Prelude hiding (filter, head)
7078import URI.ByteString hiding (Query)
7179
@@ -160,6 +168,16 @@ joinPath ps = do
160168 Server s <- bhServer <$> getBHEnv
161169 return $ joinPath' (s:ps)
162170
171+-- | Severely dumbed down query renderer. Assumes your data doesn't
172+-- need any encoding
173+addQuery :: [(Text, Maybe Text)] -> Text -> Text
174+addQuery q u = u <> rendered
175+ where
176+ rendered =
177+ T.decodeUtf8 $ BB.toByteString $ NHTU.renderQueryText prependQuestionMark q
178+ prependQuestionMark = True
179+
180+
163181bindM2 :: (Applicative m, Monad m) => (a -> b -> m c) -> m a -> m b -> m c
164182bindM2 f ma mb = join (f <$> ma <*> mb)
165183
@@ -315,15 +333,25 @@ deleteMapping (IndexName indexName)
315333-- convert into a JSON 'Value'. The 'DocId' will function as the
316334-- primary key for the document.
317335--
318−-- >>> resp <- runBH' $ indexDocument testIndex testMapping exampleTweet (DocId "1")
336+-- >>> resp <- runBH' $ indexDocument testIndex testMapping defaultIndexDocumentSettings exampleTweet (DocId "1")
319337-- >>> print resp
320338-- Response {responseStatus = Status {statusCode = 201, statusMessage = "Created"}, responseVersion = HTTP/1.1, responseHeaders = [("Content-Type","application/json; charset=UTF-8"),("Content-Length","74")], responseBody = "{\"_index\":\"twitter\",\"_type\":\"tweet\",\"_id\":\"1\",\"_version\":1,\"created\":true}", responseCookieJar = CJ {expose = []}, responseClose' = ResponseClose}
321339indexDocument :: (ToJSON doc, MonadBH m) => IndexName -> MappingName
322− -> doc -> DocId -> m Reply
340+ -> IndexDocumentSettings -> doc -> DocId -> m Reply
323341indexDocument (IndexName indexName)
324− (MappingName mappingName) document (DocId docId) =
342+ (MappingName mappingName) cfg document (DocId docId) =
325343 bindM2 put url (return body)
326− where url = joinPath [indexName, mappingName, docId]
344+ where url = addQuery params <$> joinPath [indexName, mappingName, docId]
345+ params = case idsVersionControl cfg of
346+ NoVersionControl -> []
347+ InternalVersion v -> versionParams v "internal"
348+ ExternalGT (ExternalDocVersion v) -> versionParams v "external_gt"
349+ ExternalGTE (ExternalDocVersion v) -> versionParams v "external_gte"
350+ ForceVersion (ExternalDocVersion v) -> versionParams v "force"
351+ vt = T.pack . show . docVersionNumber
352+ versionParams v t = [ ("version", Just $ vt v)
353+ , ("version_type", Just t)
354+ ]
327355 body = Just (encode document)
328356
329357-- | 'deleteDocument' is the primary way to delete a single document.
@@ -536,3 +564,17 @@ setURI req URI{..} = do
536564 Scheme "https" -> True
537565 _ -> False
538566 theQueryString = [(k , Just v) | (k, v) <- queryPairs uriQuery]
567+
568+-- | Was there an optimistic concurrency control conflict when
569+-- indexing a document?
570+isVersionConflict :: Reply -> Bool
571+isVersionConflict = statusCheck (== 409)
572+
573+isSuccess :: Reply -> Bool
574+isSuccess = statusCheck (inRange (200, 299))
575+
576+isCreated :: Reply -> Bool
577+isCreated = statusCheck (== 201)
578+
579+statusCheck :: (Int -> Bool) -> Reply -> Bool
580+statusCheck prd = prd . NHTS.statusCode . responseStatus
src/Database/Bloodhound/Types.hs+101−1
@@ -29,6 +29,7 @@
2929module Database.Bloodhound.Types
3030 ( defaultCache
3131 , defaultIndexSettings
32+ , defaultIndexDocumentSettings
3233 , mkSort
3334 , showText
3435 , unpackId
@@ -41,6 +42,8 @@ module Database.Bloodhound.Types
4142 , mkTermsAggregation
4243 , mkTermsScriptAggregation
4344 , mkDateHistogram
45+ , mkDocVersion
46+ , docVersionNumber
4447 , toTerms
4548 , toDateHistogram
4649 , omitNulls
@@ -56,6 +59,10 @@ module Database.Bloodhound.Types
5659 , Server(..)
5760 , Reply
5861 , EsResult(..)
62+ , DocVersion
63+ , ExternalDocVersion(..)
64+ , VersionControl(..)
65+ , IndexDocumentSettings(..)
5966 , Query(..)
6067 , Search(..)
6168 , SearchResult(..)
@@ -218,10 +225,12 @@ import qualified Data.ByteString.Lazy.Char8 as L
218225import Data.List (nub)
219226import Data.List.NonEmpty (NonEmpty (..), toList)
220227import qualified Data.Map.Strict as M
228+import Data.Maybe
221229import Data.Text (Text)
222230import qualified Data.Text as T
223231import Data.Time.Clock (UTCTime)
224232import qualified Data.Vector as V
233+import GHC.Enum
225234import GHC.Generics (Generic)
226235import Network.HTTP.Client
227236import qualified Network.HTTP.Types.Method as NHTM
@@ -371,10 +380,78 @@ data BulkOperation =
371380data EsResult a = EsResult { _index :: Text
372381 , _type :: Text
373382 , _id :: Text
374− , _version :: Int
383+ , _version :: DocVersion
375384 , found :: Maybe Bool
376385 , _source :: a } deriving (Eq, Show)
377386
387+{-| 'DocVersion' is an integer version number for a document between 1
388+and 9.2e+18 used for <<https://www.elastic.co/guide/en/elasticsearch/guide/current/optimistic-concurrency-control.html optimistic concurrency control>>.
389+-}
390+newtype DocVersion = DocVersion {
391+ docVersionNumber :: Int
392+ } deriving (Eq, Show, Ord, ToJSON)
393+
394+-- | Smart constructor for in-range doc version
395+mkDocVersion :: Int -> Maybe DocVersion
396+mkDocVersion i
397+ | i >= (docVersionNumber minBound) && i <= (docVersionNumber maxBound) =
398+ Just $ DocVersion i
399+ | otherwise = Nothing
400+
401+
402+{-| 'ExternalDocVersion' is a convenience wrapper if your code uses its
403+own version numbers instead of ones from ES.
404+-}
405+newtype ExternalDocVersion = ExternalDocVersion DocVersion
406+ deriving (Eq, Show, Ord, Bounded, Enum, ToJSON)
407+
408+{-| 'VersionControl' is specified when indexing documents as a
409+optimistic concurrency control.
410+-}
411+data VersionControl = NoVersionControl
412+ -- ^ Don't send a version. This is a pure overwrite.
413+ | InternalVersion DocVersion
414+ -- ^ Use the default ES versioning scheme. Only
415+ -- index the document if the version is the same
416+ -- as the one specified. Only applicable to
417+ -- updates, as you should be getting Version from
418+ -- a search result.
419+ | ExternalGT ExternalDocVersion
420+ -- ^ Use your own version numbering. Only index
421+ -- the document if the version is strictly higher
422+ -- OR the document doesn't exist. The given
423+ -- version will be used as the new version number
424+ -- for the stored document. N.B. All updates must
425+ -- increment this number, meaning there is some
426+ -- global, external ordering of updates.
427+ | ExternalGTE ExternalDocVersion
428+ -- ^ Use your own version numbering. Only index
429+ -- the document if the version is equal or higher
430+ -- than the stored version. Will succeed if there
431+ -- is no existing document. The given version will
432+ -- be used as the new version number for the
433+ -- stored document. Use with care, as this could
434+ -- result in data loss.
435+ | ForceVersion ExternalDocVersion
436+ -- ^ The document will always be indexed and the
437+ -- given version will be the new version. This is
438+ -- typically used for correcting errors. Use with
439+ -- care, as this could result in data loss.
440+ deriving (Show, Eq, Ord)
441+
442+{-| 'IndexDocumentSettings' are special settings supplied when indexing
443+a document. For the best backwards compatiblity when new fields are
444+added, you should probably prefer to start with 'defaultIndexDocumentSettings'
445+-}
446+data IndexDocumentSettings = IndexDocumentSettings {
447+ idsVersionControl :: VersionControl
448+ }
449+
450+{-| Reasonable default settings. Chooses no version control.
451+-}
452+defaultIndexDocumentSettings :: IndexDocumentSettings
453+defaultIndexDocumentSettings = IndexDocumentSettings NoVersionControl
454+
378455{-| 'Sort' is a synonym for a list of 'SortSpec's. Sort behavior is order
379456 dependent with later sorts acting as tie-breakers for earlier sorts.
380457-}
@@ -2173,3 +2250,26 @@ instance FromJSON ShardResult where
21732250 v .: "successful" <*>
21742251 v .: "failed"
21752252 parseJSON _ = empty
2253+
2254+
2255+instance FromJSON DocVersion where
2256+ parseJSON v = do
2257+ i <- parseJSON v
2258+ maybe (fail "DocVersion out of range") return $ mkDocVersion i
2259+
2260+instance Bounded DocVersion where
2261+ minBound = DocVersion 1
2262+ maxBound = DocVersion 9200000000000000000 -- 9.2e+18
2263+
2264+instance Enum DocVersion where
2265+ succ x
2266+ | x /= maxBound = DocVersion (succ $ docVersionNumber x)
2267+ | otherwise = succError "DocVersion"
2268+ pred x
2269+ | x /= minBound = DocVersion (pred $ docVersionNumber x)
2270+ | otherwise = predError "DocVersion"
2271+ toEnum i =
2272+ fromMaybe (error $ show i ++ " out of DocVersion range") $ mkDocVersion i
2273+ fromEnum = docVersionNumber
2274+ enumFrom = boundedEnumFrom
2275+ enumFromThen = boundedEnumFromThen
tests/tests.hs+43−6
@@ -5,6 +5,7 @@
55module Main where
66
77import Control.Applicative
8+import Control.Exception
89import Control.Monad
910import Control.Monad.Reader
1011import Data.Aeson
@@ -25,7 +26,7 @@ import Network.HTTP.Client
2526import qualified Network.HTTP.Types.Status as NHTS
2627import Prelude hiding (filter)
2728import Test.Hspec
28−import Test.QuickCheck.Property.Monoid
29+import Test.QuickCheck.Property.Monoid (prop_Monoid, eq, T(..))
2930
3031import Test.Hspec.QuickCheck (prop)
3132import Test.QuickCheck
@@ -147,18 +148,27 @@ otherTweet = Tweet { user = "notmyapp"
147148 , age = 1000
148149 , location = Location 40.12 (-71.34) }
149150
150−insertData :: BH IO ()
151−insertData = do
151+resetIndex :: BH IO ()
152+resetIndex = do
152153 _ <- deleteExampleIndex
153154 _ <- createExampleIndex
154155 _ <- putMapping testIndex testMapping TweetMapping
155− _ <- indexDocument testIndex testMapping exampleTweet (DocId "1")
156− _ <- refreshIndex testIndex
157156 return ()
158157
158+insertData :: BH IO Reply
159+insertData = do
160+ resetIndex
161+ insertData' defaultIndexDocumentSettings
162+
163+insertData' :: IndexDocumentSettings -> BH IO Reply
164+insertData' ids = do
165+ r <- indexDocument testIndex testMapping ids exampleTweet (DocId "1")
166+ _ <- refreshIndex testIndex
167+ return r
168+
159169insertOther :: BH IO ()
160170insertOther = do
161− _ <- indexDocument testIndex testMapping otherTweet (DocId "2")
171+ _ <- indexDocument testIndex testMapping defaultIndexDocumentSettings otherTweet (DocId "2")
162172 _ <- refreshIndex testIndex
163173 return ()
164174
@@ -290,6 +300,14 @@ main = hspec $ do
290300 (responseBody docInserted) :: Either String (EsResult Tweet)
291301 liftIO $ (fmap _source newTweet `shouldBe` Right exampleTweet)
292302
303+ it "can use optimistic concurrency control" $ withTestEnv $ do
304+ let ev = ExternalDocVersion minBound
305+ let cfg = defaultIndexDocumentSettings { idsVersionControl = ExternalGT ev }
306+ resetIndex
307+ res <- insertData' cfg
308+ liftIO $ isCreated res `shouldBe` True
309+ res' <- insertData' cfg
310+ liftIO $ isVersionConflict res' `shouldBe` True
293311
294312 describe "bulk API" $ do
295313 it "inserts all documents we request" $ withTestEnv $ do
@@ -659,3 +677,22 @@ main = hspec $ do
659677 describe "Monoid (SearchHits a)" $ do
660678 prop "abides the monoid laws" $ eq $
661679 prop_Monoid (T :: T (SearchHits ()))
680+
681+ describe "mkDocVersion" $ do
682+ prop "can never construct an out of range docVersion" $ \i ->
683+ let res = mkDocVersion i
684+ in case res of
685+ Nothing -> property True
686+ Just dv -> (dv >= minBound) .&&.
687+ (dv <= maxBound) .&&.
688+ docVersionNumber dv === i
689+
690+ describe "Enum DocVersion" $ do
691+ it "follows the laws of Enum, Bounded" $ do
692+ evaluate (succ maxBound :: DocVersion) `shouldThrow` anyErrorCall
693+ evaluate (pred minBound :: DocVersion) `shouldThrow` anyErrorCall
694+ evaluate (toEnum 0 :: DocVersion) `shouldThrow` anyErrorCall
695+ evaluate (toEnum 9200000000000000001 :: DocVersion) `shouldThrow` anyErrorCall
696+ enumFrom (pred maxBound :: DocVersion) `shouldBe` [pred maxBound, maxBound]
697+ enumFrom (pred maxBound :: DocVersion) `shouldBe` [pred maxBound, maxBound]
698+ enumFromThen minBound (pred maxBound :: DocVersion) `shouldBe` [minBound, pred maxBound]