0a667b8 · typeable/bloodhound

git.iko.soy/typeable/bloodhound

on master help sign in

Merge branch 'master' into update-index-settings

0a667b8Michael Xavier11 years ago

.travis.yml+1

compare with master

unifiedsplit

@@ -29,6 +29,7 @@ before_install:

29 - sudo apt-get update

30 - sudo apt-get install stack -y

31 - stack setup

32+ - stack update

32 - wget --no-check-certificate https://download.elasticsearch.org/elasticsearch/elasticsearch/elasticsearch-$ESVER.deb

33 - wget --no-check-certificate https://download.elasticsearch.org/elasticsearch/elasticsearch/elasticsearch-$ESVER.deb

33 - sudo dpkg --force-all -i elasticsearch-$ESVER.deb

34 - sudo dpkg --force-all -i elasticsearch-$ESVER.deb

34 - sudo service elasticsearch start

35 - sudo service elasticsearch start

bloodhound.cabal+3−3

@@ -1,5 +1,5 @@

1name: bloodhound

2−version: 0.8.0.0

2+version: 0.9.0.0

3synopsis: ElasticSearch client library for Haskell

4description: ElasticSearch made awesome for Haskell hackers

5homepage: https://github.com/bitemyapp/bloodhound

@@ -31,7 +31,7 @@ library

31 build-depends: base >= 4.3 && <5,

32 bytestring >= 0.10.0 && <0.11,

33 containers >= 0.5.0.0 && <0.6,

34− aeson >= 0.7 && <0.10,

34+ aeson >= 0.10 && <0.11,

35 http-client >= 0.3 && <0.5,

36 network-uri >= 2.6 && <2.7,

37 semigroups >= 0.15 && <0.18,

@@ -39,7 +39,7 @@ library

39 text >= 0.11 && <1.3,

40 mtl >= 1.0 && <2.3,

41 transformers >= 0.2 && <0.5,

42− http-types >= 0.8 && <0.9,

42+ http-types >= 0.8 && <0.10,

43 vector >= 0.10.9 && <0.12,

44 exceptions,

45 data-default-class,

src/Database/Bloodhound/Client.hs+12−6

@@ -123,13 +123,19 @@ import Database.Bloodhound.Types

123-- , message = "Use haskell!"

124-- , age = 10000

125-- , location = Location 40.12 (-71.34) }

126−--instance ToJSON Tweet

126+--instance ToJSON Tweet where

127+-- toJSON = genericToJSON defaultOptions

127−--instance FromJSON Tweet

128+--instance FromJSON Tweet where

129+-- parseJSON = genericParseJSON defaultOptions

128−--instance ToJSON Location

130+--instance ToJSON Location where

131+-- toJSON = genericToJSON defaultOptions

129−--instance FromJSON Location

132+--instance FromJSON Location where

133+-- parseJSON = genericParseJSON defaultOptions

130--data BulkTest = BulkTest { name :: Text } deriving (Eq, Generic, Show)

134--data BulkTest = BulkTest { name :: Text } deriving (Eq, Generic, Show)

131−--instance FromJSON BulkTest

135+--instance FromJSON BulkTest where

136+-- parseJSON = genericParseJSON defaultOptions

132−--instance ToJSON BulkTest

137+--instance ToJSON BulkTest where

138+-- toJSON = genericToJSON defaultOptions

133-- :}

139-- :}

134

140

135-- | 'mkShardCount' is a straight-forward smart constructor for 'ShardCount'

141-- | 'mkShardCount' is a straight-forward smart constructor for 'ShardCount'

src/Database/Bloodhound/Types.hs+105−50

@@ -70,6 +70,7 @@ module Database.Bloodhound.Types

70 , Reply

71 , EsResult(..)

72 , EsResultFound(..)

73+ , EsError(..)

73 , DocVersion

74 , DocVersion

74 , ExternalDocVersion(..)

75 , ExternalDocVersion(..)

75 , VersionControl(..)

76 , VersionControl(..)

@@ -493,13 +494,18 @@ data EsResult a = EsResult { _index :: Text

493 , _id :: Text

494 , _id :: Text

494 , foundResult :: Maybe (EsResultFound a)} deriving (Eq, Show)

495 , foundResult :: Maybe (EsResultFound a)} deriving (Eq, Show)

495

496

496−

497{-| 'EsResultFound' contains the document and its metadata inside of an

498 'EsResult' when the document was successfully found.

499-}

500data EsResultFound a = EsResultFound { _version :: DocVersion

501 , _source :: a } deriving (Eq, Show)

502

503+{-| 'EsError' is the generic type that will be returned when there was a

504+ problem. If you can't parse the expected response, its a good idea to

505+ try parsing this.

506+-}

507+data EsError = EsError { errorStatus :: Int

508+ , errorMessage :: Text } deriving (Eq, Show)

503

509

504

510

505{-| 'DocVersion' is an integer version number for a document between 1

511{-| 'DocVersion' is an integer version number for a document between 1

@@ -833,7 +839,7 @@ data SearchType = SearchTypeQueryThenFetch

833 deriving (Eq, Show)

839 deriving (Eq, Show)

834

840

835data Source =

841data Source =

836− NoSource

842+ NoSource

837 | SourcePatterns PatternOrPatterns

843 | SourcePatterns PatternOrPatterns

838 | SourceIncludeExclude Include Exclude

844 | SourceIncludeExclude Include Exclude

839 deriving (Show, Eq)

845 deriving (Show, Eq)

@@ -1382,7 +1388,7 @@ data Hit a =

1382 , hitType :: MappingName

1388 , hitType :: MappingName

1383 , hitDocId :: DocId

1389 , hitDocId :: DocId

1384 , hitScore :: Score

1390 , hitScore :: Score

1385− , hitSource :: a

1391+ , hitSource :: Maybe a

1386 , hitHighlight :: Maybe HitHighlight } deriving (Eq, Show)

1392 , hitHighlight :: Maybe HitHighlight } deriving (Eq, Show)

1387

1393

1388data ShardResult =

1394data ShardResult =

@@ -2120,48 +2126,91 @@ instance ToJSON MatchQueryType where

2120instance ToJSON FieldName where

2126instance ToJSON FieldName where

2121 toJSON (FieldName fieldName) = String fieldName

2127 toJSON (FieldName fieldName) = String fieldName

2122

2128

2123−instance ToJSON ReplicaCount

2129+instance ToJSON ReplicaCount where

2130+ toJSON = genericToJSON defaultOptions

2124−instance ToJSON ShardCount

2131+instance ToJSON ShardCount where

2132+ toJSON = genericToJSON defaultOptions

2125−instance ToJSON CutoffFrequency

2133+instance ToJSON CutoffFrequency where

2134+ toJSON = genericToJSON defaultOptions

2126−instance ToJSON Analyzer

2135+instance ToJSON Analyzer where

2136+ toJSON = genericToJSON defaultOptions

2127−instance ToJSON MaxExpansions

2137+instance ToJSON MaxExpansions where

2138+ toJSON = genericToJSON defaultOptions

2128−instance ToJSON Lenient

2139+instance ToJSON Lenient where

2140+ toJSON = genericToJSON defaultOptions

2129−instance ToJSON Boost

2141+instance ToJSON Boost where

2142+ toJSON = genericToJSON defaultOptions

2130−instance ToJSON Version

2143+instance ToJSON Version where

2144+ toJSON = genericToJSON defaultOptions

2131−instance ToJSON Tiebreaker

2145+instance ToJSON Tiebreaker where

2146+ toJSON = genericToJSON defaultOptions

2132−instance ToJSON MinimumMatch

2147+instance ToJSON MinimumMatch where

2148+ toJSON = genericToJSON defaultOptions

2133−instance ToJSON DisableCoord

2149+instance ToJSON DisableCoord where

2150+ toJSON = genericToJSON defaultOptions

2134−instance ToJSON PrefixLength

2151+instance ToJSON PrefixLength where

2152+ toJSON = genericToJSON defaultOptions

2135−instance ToJSON Fuzziness

2153+instance ToJSON Fuzziness where

2154+ toJSON = genericToJSON defaultOptions

2136−instance ToJSON IgnoreTermFrequency

2155+instance ToJSON IgnoreTermFrequency where

2156+ toJSON = genericToJSON defaultOptions

2137−instance ToJSON MaxQueryTerms

2157+instance ToJSON MaxQueryTerms where

2158+ toJSON = genericToJSON defaultOptions

2138−instance ToJSON TypeName

2159+instance ToJSON TypeName where

2160+ toJSON = genericToJSON defaultOptions

2139−instance ToJSON IndexName

2161+instance ToJSON IndexName where

2162+ toJSON = genericToJSON defaultOptions

2140−instance ToJSON TemplateName

2163+instance ToJSON TemplateName where

2164+ toJSON = genericToJSON defaultOptions

2141−instance ToJSON TemplatePattern

2165+instance ToJSON TemplatePattern where

2166+ toJSON = genericToJSON defaultOptions

2142−instance ToJSON BoostTerms

2167+instance ToJSON BoostTerms where

2168+ toJSON = genericToJSON defaultOptions

2143−instance ToJSON MaxWordLength

2169+instance ToJSON MaxWordLength where

2170+ toJSON = genericToJSON defaultOptions

2144−instance ToJSON MinWordLength

2171+instance ToJSON MinWordLength where

2172+ toJSON = genericToJSON defaultOptions

2145−instance ToJSON MaxDocFrequency

2173+instance ToJSON MaxDocFrequency where

2174+ toJSON = genericToJSON defaultOptions

2146−instance ToJSON MinDocFrequency

2175+instance ToJSON MinDocFrequency where

2176+ toJSON = genericToJSON defaultOptions

2147−instance ToJSON PhraseSlop

2177+instance ToJSON PhraseSlop where

2178+ toJSON = genericToJSON defaultOptions

2148−instance ToJSON StopWord

2179+instance ToJSON StopWord where

2180+ toJSON = genericToJSON defaultOptions

2149−instance ToJSON QueryPath

2181+instance ToJSON QueryPath where

2182+ toJSON = genericToJSON defaultOptions

2150−instance ToJSON MinimumTermFrequency

2183+instance ToJSON MinimumTermFrequency where

2184+ toJSON = genericToJSON defaultOptions

2151−instance ToJSON PercentMatch

2185+instance ToJSON PercentMatch where

2186+ toJSON = genericToJSON defaultOptions

2152−instance ToJSON MappingName

2187+instance ToJSON MappingName where

2188+ toJSON = genericToJSON defaultOptions

2153−instance ToJSON DocId

2189+instance ToJSON DocId where

2190+ toJSON = genericToJSON defaultOptions

2154−instance ToJSON QueryString

2191+instance ToJSON QueryString where

2192+ toJSON = genericToJSON defaultOptions

2155−instance ToJSON AllowLeadingWildcard

2193+instance ToJSON AllowLeadingWildcard where

2194+ toJSON = genericToJSON defaultOptions

2156−instance ToJSON LowercaseExpanded

2195+instance ToJSON LowercaseExpanded where

2196+ toJSON = genericToJSON defaultOptions

2157−instance ToJSON AnalyzeWildcard

2197+instance ToJSON AnalyzeWildcard where

2198+ toJSON = genericToJSON defaultOptions

2158−instance ToJSON GeneratePhraseQueries

2199+instance ToJSON GeneratePhraseQueries where

2200+ toJSON = genericToJSON defaultOptions

2159−instance ToJSON Locale

2201+instance ToJSON Locale where

2202+ toJSON = genericToJSON defaultOptions

2160−instance ToJSON EnablePositionIncrements

2203+instance ToJSON EnablePositionIncrements where

2204+ toJSON = genericToJSON defaultOptions

2205+

2161−instance FromJSON Version

2206+instance FromJSON Version where

2207+ parseJSON = genericParseJSON defaultOptions

2162−instance FromJSON IndexName

2208+instance FromJSON IndexName where

2209+ parseJSON = genericParseJSON defaultOptions

2163−instance FromJSON MappingName

2210+instance FromJSON MappingName where

2211+ parseJSON = genericParseJSON defaultOptions

2164−instance FromJSON DocId

2212+instance FromJSON DocId where

2213+ parseJSON = genericParseJSON defaultOptions

2165

2214

2166

2215

2167instance FromJSON Status where

2216instance FromJSON Status where

@@ -2280,6 +2329,12 @@ instance (FromJSON a) => FromJSON (EsResultFound a) where

2280 v .: "_source"

2329 v .: "_source"

2281 parseJSON _ = empty

2330 parseJSON _ = empty

2282

2331

2332+instance FromJSON EsError where

2333+ parseJSON (Object v) = EsError <$>

2334+ v .: "status" <*>

2335+ v .: "error"

2336+ parseJSON _ = empty

2337+

2283instance ToJSON Search where

2338instance ToJSON Search where

2284 toJSON (Search query sFilter sort searchAggs highlight sTrackSortScores sFrom sSize _ sFields sSource) =

2339 toJSON (Search query sFilter sort searchAggs highlight sTrackSortScores sFrom sSize _ sFields sSource) =

2285 omitNulls [ "query" .= query

2340 omitNulls [ "query" .= query

@@ -2543,11 +2598,11 @@ instance (FromJSON a) => FromJSON (SearchHits a) where

2543

2598

2544instance (FromJSON a) => FromJSON (Hit a) where

2599instance (FromJSON a) => FromJSON (Hit a) where

2545 parseJSON (Object v) = Hit <$>

2600 parseJSON (Object v) = Hit <$>

2546− v .: "_index" <*>

2601+ v .: "_index" <*>

2547− v .: "_type" <*>

2602+ v .: "_type" <*>

2548− v .: "_id" <*>

2603+ v .: "_id" <*>

2549− v .: "_score" <*>

2604+ v .: "_score" <*>

2550− v .: "_source" <*>

2605+ v .:? "_source" <*>

2551 v .:? "highlight"

2606 v .:? "highlight"

2552 parseJSON _ = empty

2607 parseJSON _ = empty

2553

2608

stack-7.10.yaml+6−1

@@ -2,7 +2,12 @@ flags: {}

2packages:

3- '.'

4extra-deps:

5+- aeson-0.10.0.0

6+- http-types-0.9

7+- http-client-0.4.24

8+- attoparsec-0.13.0.1

5- doctest-0.10.1

9- doctest-0.10.1

6- doctest-prop-0.2.0.1

10- doctest-prop-0.2.0.1

7- quickcheck-properties-0.1

11- quickcheck-properties-0.1

12+- uri-bytestring-0.1.9

8−resolver: lts-3.1

13+resolver: lts-3.10

stack-7.8.yamlmode 120000 → 100644+12−1

@@ -1 +0,0 @@

1−stack.yaml

@@ -0,0 +1,12 @@

1+flags: {}

2+packages:

3+- '.'

4+extra-deps:

5+- aeson-0.10.0.0

6+- http-types-0.9

7+- http-client-0.4.24

8+- attoparsec-0.13.0.1

9+- doctest-0.10.1

10+- doctest-prop-0.2.0.1

11+- quickcheck-properties-0.1

12+resolver: lts-2.18

stack.yamlmode 100644 → 120000+1−8

@@ -1,8 +0,0 @@

1−flags: {}

2−packages:

3−- '.'

4−extra-deps:

5−- doctest-0.10.1

6−- doctest-prop-0.2.0.1

7−- quickcheck-properties-0.1

8−resolver: lts-2.18

@@ -0,0 +1 @@

1+stack-7.10.yaml

tests/tests.hs+39−25

@@ -14,6 +14,7 @@ import Data.List (nub)

14import Data.List.NonEmpty (NonEmpty (..))

15import qualified Data.List.NonEmpty as NE

16import qualified Data.Map.Strict as M

17+import qualified Data.Maybe as MY

17import Data.Monoid

18import Data.Monoid

18import Data.Text (Text)

19import Data.Text (Text)

19import qualified Data.Text as T

20import qualified Data.Text as T

@@ -27,7 +28,7 @@ import Network.HTTP.Client

27import qualified Network.HTTP.Types.Status as NHTS

28import qualified Network.HTTP.Types.Status as NHTS

28import Prelude hiding (filter)

29import Prelude hiding (filter)

29import Test.Hspec

30import Test.Hspec

30−import Test.QuickCheck.Property.Monoid (prop_Monoid, eq, T(..))

31+import Test.QuickCheck.Property.Monoid (T (..), eq, prop_Monoid)

32import Test.Hspec.QuickCheck (prop)

33import Test.Hspec.QuickCheck (prop)

33import Test.QuickCheck

34import Test.QuickCheck

@@ -196,7 +197,8 @@ insertWithSpaceInId = do

196searchTweet :: Search -> BH IO (Either String Tweet)

197searchTweet :: Search -> BH IO (Either String Tweet)

197searchTweet search = do

198searchTweet search = do

198 result <- searchTweets search

199 result <- searchTweets search

199− let myTweet = fmap (hitSource . head . hits . searchHits) result

200+ let myTweet :: Either String Tweet

201+ myTweet = grabFirst result

200 return myTweet

202 return myTweet

201

203

202searchTweets :: Search -> BH IO (Either String (SearchResult Tweet))

204searchTweets :: Search -> BH IO (Either String (SearchResult Tweet))

@@ -247,7 +249,7 @@ searchExpectSource src expected = do

247 let search = (mkSearch (Just query) Nothing) { source = Just src }

249 let search = (mkSearch (Just query) Nothing) { source = Just src }

248 reply <- searchAll search

250 reply <- searchAll search

249 let result = eitherDecode (responseBody reply) :: Either String (SearchResult Value)

251 let result = eitherDecode (responseBody reply) :: Either String (SearchResult Value)

250− let value = fmap (hitSource . head . hits . searchHits) result

252+ let value = grabFirst result

251 liftIO $

253 liftIO $

252 value `shouldBe` expected

254 value `shouldBe` expected

253

255

@@ -313,6 +315,13 @@ instance Arbitrary a => Arbitrary (SearchHits a) where

313getSource :: EsResult a -> Maybe a

315getSource :: EsResult a -> Maybe a

314getSource = fmap _source . foundResult

316getSource = fmap _source . foundResult

315

317

318+-- grabFirst :: Either String (SearchResult a) -> Either String a

319+grabFirst r =

320+ case fmap (hitSource . head . hits . searchHits) r of

321+ (Left e) -> Left e

322+ (Right Nothing) -> Left "Source was missing"

323+ (Right (Just x)) -> Right x

324+

316main :: IO ()

325main :: IO ()

317main = hspec $ do

326main = hspec $ do

318

327

@@ -326,6 +335,11 @@ main = hspec $ do

326 validateStatus resp 200

335 validateStatus resp 200

327 validateStatus deleteResp 200

336 validateStatus deleteResp 200

328

337

338+ describe "error parsing" $ do

339+ it "can parse EsErrors" $ withTestEnv $ do

340+ res <- getDocument (IndexName "bogus") (MappingName "also_bogus") (DocId "bogus_as_well")

341+ let errorResp = eitherDecode (responseBody res)

342+ liftIO (errorResp `shouldBe` Right (EsError 404 "IndexMissingException[[bogus] missing]"))

329

343

330 describe "document API" $ do

344 describe "document API" $ do

331 it "indexes, gets, and then deletes the generated document" $ withTestEnv $ do

345 it "indexes, gets, and then deletes the generated document" $ withTestEnv $ do

@@ -487,7 +501,7 @@ main = hspec $ do

487 (Just IdentityFilter) (Just [sortSpec]) Nothing Nothing

501 (Just IdentityFilter) (Just [sortSpec]) Nothing Nothing

488 False (From 0) (Size 10) SearchTypeQueryThenFetch Nothing Nothing

502 False (From 0) (Size 10) SearchTypeQueryThenFetch Nothing Nothing

489 result <- searchTweets search

503 result <- searchTweets search

490− let myTweet = fmap (hitSource . head . hits . searchHits) result

504+ let myTweet = grabFirst result

491 liftIO $

505 liftIO $

492 myTweet `shouldBe` Right otherTweet

506 myTweet `shouldBe` Right otherTweet

493

507

@@ -704,25 +718,25 @@ main = hspec $ do

704 fmap aggregations res `shouldBe` Right (Just (M.fromList [ docCountPair "bitemyapps" 1

718 fmap aggregations res `shouldBe` Right (Just (M.fromList [ docCountPair "bitemyapps" 1

705 , docCountPair "notmyapps" 1

719 , docCountPair "notmyapps" 1

706 ]))

720 ]))

707− -- Interaction of date serialization and date histogram aggregation is broken.

721+

708− -- it "returns date histogram aggregation results" $ withTestEnv $ do

722+ it "returns date histogram aggregation results" $ withTestEnv $ do

709− -- _ <- insertData

723+ _ <- insertData

710− -- let histogram = DateHistogramAgg $ mkDateHistogram (FieldName "postDate") Minute

724+ let histogram = DateHistogramAgg $ mkDateHistogram (FieldName "postDate") Minute

711− -- let search = mkAggregateSearch Nothing (mkAggregations "byDate" histogram)

725+ let search = mkAggregateSearch Nothing (mkAggregations "byDate" histogram)

712− -- searchExpectAggs search

726+ searchExpectAggs search

713− -- searchValidBucketAgg search "byDate" toDateHistogram

727+ searchValidBucketAgg search "byDate" toDateHistogram

714−

728+

715− -- it "returns date histogram using fractional date" $ withTestEnv $ do

729+ it "returns date histogram using fractional date" $ withTestEnv $ do

716− -- _ <- insertData

730+ _ <- insertData

717− -- let periods = [Year, Quarter, Month, Week, Day, Hour, Minute, Second]

731+ let periods = [Year, Quarter, Month, Week, Day, Hour, Minute, Second]

718− -- let fractionals = map (FractionalInterval 1.5) [Weeks, Days, Hours, Minutes, Seconds]

732+ let fractionals = map (FractionalInterval 1.5) [Weeks, Days, Hours, Minutes, Seconds]

719− -- let intervals = periods ++ fractionals

733+ let intervals = periods ++ fractionals

720− -- let histogram = mkDateHistogram (FieldName "postDate")

734+ let histogram = mkDateHistogram (FieldName "postDate")

721− -- let search interval = mkAggregateSearch Nothing $ mkAggregations "byDate" $ DateHistogramAgg (histogram interval)

735+ let search interval = mkAggregateSearch Nothing $ mkAggregations "byDate" $ DateHistogramAgg (histogram interval)

722− -- let expect interval = searchExpectAggs (search interval)

736+ let expect interval = searchExpectAggs (search interval)

723− -- let valid interval = searchValidBucketAgg (search interval) "byDate" toDateHistogram

737+ let valid interval = searchValidBucketAgg (search interval) "byDate" toDateHistogram

724− -- forM_ intervals expect

738+ forM_ intervals expect

725− -- forM_ intervals valid

739+ forM_ intervals valid

726

740

727 describe "Highlights API" $ do

741 describe "Highlights API" $ do

728

742

@@ -753,7 +767,7 @@ main = hspec $ do

753 it "doesn't include source when sources are disabled" $ withTestEnv $ do

767 it "doesn't include source when sources are disabled" $ withTestEnv $ do

754 searchExpectSource

768 searchExpectSource

755 NoSource

769 NoSource

756− (Left "key \"_source\" not present")

770+ (Left "Source was missing")

757

771

758 it "includes a source" $ withTestEnv $ do

772 it "includes a source" $ withTestEnv $ do

759 searchExpectSource

773 searchExpectSource

@@ -849,4 +863,4 @@ main = hspec $ do

849 liftIO $

863 liftIO $

850 regular_search `shouldBe` Right exampleTweet -- Check that the size restrtiction is being honored

864 regular_search `shouldBe` Right exampleTweet -- Check that the size restrtiction is being honored

851 liftIO $

865 liftIO $

852− scan_search `shouldMatchList` [exampleTweet, otherTweet]

866+ scan_search `shouldMatchList` [Just exampleTweet, Just otherTweet]