0a667b8 · typeable/bloodhound

git.iko.soy/typeable/bloodhound

Merge branch 'master' into update-index-settings

commit0a667b820ef20f948bb13d1c6a4340f67f8034b0 authorMichael Xavier <michael@michaelxavier.net> dateSun 18 Oct 2015, 20:17 (11 years ago) parentb0ba889 a971422

8 files changed+179−94

compare with master

unifiedsplit

.travis.yml+1 bloodhound.cabal+3−3 src/Database/Bloodhound/Client.hs+12−6 src/Database/Bloodhound/Types.hs+105−50 stack-7.10.yaml+6−1 stack-7.8.yaml+12−1 stack.yaml+1−8 tests/tests.hs+39−25

.travis.yml+1

@@ -29,6 +29,7 @@ before_install:

2929 - sudo apt-get update

3030 - sudo apt-get install stack -y

3131 - stack setup

32+ - stack update

3233 - wget --no-check-certificate https://download.elasticsearch.org/elasticsearch/elasticsearch/elasticsearch-$ESVER.deb

3334 - sudo dpkg --force-all -i elasticsearch-$ESVER.deb

3435 - sudo service elasticsearch start

bloodhound.cabal+3−3

@@ -1,5 +1,5 @@

11name: bloodhound

2−version: 0.8.0.0

2+version: 0.9.0.0

33synopsis: ElasticSearch client library for Haskell

44description: ElasticSearch made awesome for Haskell hackers

55homepage: https://github.com/bitemyapp/bloodhound

@@ -31,7 +31,7 @@ library

3131 build-depends: base >= 4.3 && <5,

3232 bytestring >= 0.10.0 && <0.11,

3333 containers >= 0.5.0.0 && <0.6,

34− aeson >= 0.7 && <0.10,

34+ aeson >= 0.10 && <0.11,

3535 http-client >= 0.3 && <0.5,

3636 network-uri >= 2.6 && <2.7,

3737 semigroups >= 0.15 && <0.18,

@@ -39,7 +39,7 @@ library

3939 text >= 0.11 && <1.3,

4040 mtl >= 1.0 && <2.3,

4141 transformers >= 0.2 && <0.5,

42− http-types >= 0.8 && <0.9,

42+ http-types >= 0.8 && <0.10,

4343 vector >= 0.10.9 && <0.12,

4444 exceptions,

4545 data-default-class,

src/Database/Bloodhound/Client.hs+12−6

@@ -123,13 +123,19 @@ import Database.Bloodhound.Types

123123-- , message = "Use haskell!"

124124-- , age = 10000

125125-- , location = Location 40.12 (-71.34) }

126−--instance ToJSON Tweet

127−--instance FromJSON Tweet

128−--instance ToJSON Location

129−--instance FromJSON Location

126+--instance ToJSON Tweet where

127+-- toJSON = genericToJSON defaultOptions

128+--instance FromJSON Tweet where

129+-- parseJSON = genericParseJSON defaultOptions

130+--instance ToJSON Location where

131+-- toJSON = genericToJSON defaultOptions

132+--instance FromJSON Location where

133+-- parseJSON = genericParseJSON defaultOptions

130134--data BulkTest = BulkTest { name :: Text } deriving (Eq, Generic, Show)

131−--instance FromJSON BulkTest

132−--instance ToJSON BulkTest

135+--instance FromJSON BulkTest where

136+-- parseJSON = genericParseJSON defaultOptions

137+--instance ToJSON BulkTest where

138+-- toJSON = genericToJSON defaultOptions

133139-- :}

134140

135141-- | 'mkShardCount' is a straight-forward smart constructor for 'ShardCount'

src/Database/Bloodhound/Types.hs+105−50

@@ -70,6 +70,7 @@ module Database.Bloodhound.Types

7070 , Reply

7171 , EsResult(..)

7272 , EsResultFound(..)

73+ , EsError(..)

7374 , DocVersion

7475 , ExternalDocVersion(..)

7576 , VersionControl(..)

@@ -493,13 +494,18 @@ data EsResult a = EsResult { _index :: Text

493494 , _id :: Text

494495 , foundResult :: Maybe (EsResultFound a)} deriving (Eq, Show)

495496

496−

497497{-| 'EsResultFound' contains the document and its metadata inside of an

498498 'EsResult' when the document was successfully found.

499499-}

500500data EsResultFound a = EsResultFound { _version :: DocVersion

501501 , _source :: a } deriving (Eq, Show)

502502

503+{-| 'EsError' is the generic type that will be returned when there was a

504+ problem. If you can't parse the expected response, its a good idea to

505+ try parsing this.

506+-}

507+data EsError = EsError { errorStatus :: Int

508+ , errorMessage :: Text } deriving (Eq, Show)

503509

504510

505511{-| 'DocVersion' is an integer version number for a document between 1

@@ -833,7 +839,7 @@ data SearchType = SearchTypeQueryThenFetch

833839 deriving (Eq, Show)

834840

835841data Source =

836− NoSource

842+ NoSource

837843 | SourcePatterns PatternOrPatterns

838844 | SourceIncludeExclude Include Exclude

839845 deriving (Show, Eq)

@@ -1382,7 +1388,7 @@ data Hit a =

13821388 , hitType :: MappingName

13831389 , hitDocId :: DocId

13841390 , hitScore :: Score

1385− , hitSource :: a

1391+ , hitSource :: Maybe a

13861392 , hitHighlight :: Maybe HitHighlight } deriving (Eq, Show)

13871393

13881394data ShardResult =

@@ -2120,48 +2126,91 @@ instance ToJSON MatchQueryType where

21202126instance ToJSON FieldName where

21212127 toJSON (FieldName fieldName) = String fieldName

21222128

2123−instance ToJSON ReplicaCount

2124−instance ToJSON ShardCount

2125−instance ToJSON CutoffFrequency

2126−instance ToJSON Analyzer

2127−instance ToJSON MaxExpansions

2128−instance ToJSON Lenient

2129−instance ToJSON Boost

2130−instance ToJSON Version

2131−instance ToJSON Tiebreaker

2132−instance ToJSON MinimumMatch

2133−instance ToJSON DisableCoord

2134−instance ToJSON PrefixLength

2135−instance ToJSON Fuzziness

2136−instance ToJSON IgnoreTermFrequency

2137−instance ToJSON MaxQueryTerms

2138−instance ToJSON TypeName

2139−instance ToJSON IndexName

2140−instance ToJSON TemplateName

2141−instance ToJSON TemplatePattern

2142−instance ToJSON BoostTerms

2143−instance ToJSON MaxWordLength

2144−instance ToJSON MinWordLength

2145−instance ToJSON MaxDocFrequency

2146−instance ToJSON MinDocFrequency

2147−instance ToJSON PhraseSlop

2148−instance ToJSON StopWord

2149−instance ToJSON QueryPath

2150−instance ToJSON MinimumTermFrequency

2151−instance ToJSON PercentMatch

2152−instance ToJSON MappingName

2153−instance ToJSON DocId

2154−instance ToJSON QueryString

2155−instance ToJSON AllowLeadingWildcard

2156−instance ToJSON LowercaseExpanded

2157−instance ToJSON AnalyzeWildcard

2158−instance ToJSON GeneratePhraseQueries

2159−instance ToJSON Locale

2160−instance ToJSON EnablePositionIncrements

2161−instance FromJSON Version

2162−instance FromJSON IndexName

2163−instance FromJSON MappingName

2164−instance FromJSON DocId

2129+instance ToJSON ReplicaCount where

2130+ toJSON = genericToJSON defaultOptions

2131+instance ToJSON ShardCount where

2132+ toJSON = genericToJSON defaultOptions

2133+instance ToJSON CutoffFrequency where

2134+ toJSON = genericToJSON defaultOptions

2135+instance ToJSON Analyzer where

2136+ toJSON = genericToJSON defaultOptions

2137+instance ToJSON MaxExpansions where

2138+ toJSON = genericToJSON defaultOptions

2139+instance ToJSON Lenient where

2140+ toJSON = genericToJSON defaultOptions

2141+instance ToJSON Boost where

2142+ toJSON = genericToJSON defaultOptions

2143+instance ToJSON Version where

2144+ toJSON = genericToJSON defaultOptions

2145+instance ToJSON Tiebreaker where

2146+ toJSON = genericToJSON defaultOptions

2147+instance ToJSON MinimumMatch where

2148+ toJSON = genericToJSON defaultOptions

2149+instance ToJSON DisableCoord where

2150+ toJSON = genericToJSON defaultOptions

2151+instance ToJSON PrefixLength where

2152+ toJSON = genericToJSON defaultOptions

2153+instance ToJSON Fuzziness where

2154+ toJSON = genericToJSON defaultOptions

2155+instance ToJSON IgnoreTermFrequency where

2156+ toJSON = genericToJSON defaultOptions

2157+instance ToJSON MaxQueryTerms where

2158+ toJSON = genericToJSON defaultOptions

2159+instance ToJSON TypeName where

2160+ toJSON = genericToJSON defaultOptions

2161+instance ToJSON IndexName where

2162+ toJSON = genericToJSON defaultOptions

2163+instance ToJSON TemplateName where

2164+ toJSON = genericToJSON defaultOptions

2165+instance ToJSON TemplatePattern where

2166+ toJSON = genericToJSON defaultOptions

2167+instance ToJSON BoostTerms where

2168+ toJSON = genericToJSON defaultOptions

2169+instance ToJSON MaxWordLength where

2170+ toJSON = genericToJSON defaultOptions

2171+instance ToJSON MinWordLength where

2172+ toJSON = genericToJSON defaultOptions

2173+instance ToJSON MaxDocFrequency where

2174+ toJSON = genericToJSON defaultOptions

2175+instance ToJSON MinDocFrequency where

2176+ toJSON = genericToJSON defaultOptions

2177+instance ToJSON PhraseSlop where

2178+ toJSON = genericToJSON defaultOptions

2179+instance ToJSON StopWord where

2180+ toJSON = genericToJSON defaultOptions

2181+instance ToJSON QueryPath where

2182+ toJSON = genericToJSON defaultOptions

2183+instance ToJSON MinimumTermFrequency where

2184+ toJSON = genericToJSON defaultOptions

2185+instance ToJSON PercentMatch where

2186+ toJSON = genericToJSON defaultOptions

2187+instance ToJSON MappingName where

2188+ toJSON = genericToJSON defaultOptions

2189+instance ToJSON DocId where

2190+ toJSON = genericToJSON defaultOptions

2191+instance ToJSON QueryString where

2192+ toJSON = genericToJSON defaultOptions

2193+instance ToJSON AllowLeadingWildcard where

2194+ toJSON = genericToJSON defaultOptions

2195+instance ToJSON LowercaseExpanded where

2196+ toJSON = genericToJSON defaultOptions

2197+instance ToJSON AnalyzeWildcard where

2198+ toJSON = genericToJSON defaultOptions

2199+instance ToJSON GeneratePhraseQueries where

2200+ toJSON = genericToJSON defaultOptions

2201+instance ToJSON Locale where

2202+ toJSON = genericToJSON defaultOptions

2203+instance ToJSON EnablePositionIncrements where

2204+ toJSON = genericToJSON defaultOptions

2205+

2206+instance FromJSON Version where

2207+ parseJSON = genericParseJSON defaultOptions

2208+instance FromJSON IndexName where

2209+ parseJSON = genericParseJSON defaultOptions

2210+instance FromJSON MappingName where

2211+ parseJSON = genericParseJSON defaultOptions

2212+instance FromJSON DocId where

2213+ parseJSON = genericParseJSON defaultOptions

21652214

21662215

21672216instance FromJSON Status where

@@ -2280,6 +2329,12 @@ instance (FromJSON a) => FromJSON (EsResultFound a) where

22802329 v .: "_source"

22812330 parseJSON _ = empty

22822331

2332+instance FromJSON EsError where

2333+ parseJSON (Object v) = EsError <$>

2334+ v .: "status" <*>

2335+ v .: "error"

2336+ parseJSON _ = empty

2337+

22832338instance ToJSON Search where

22842339 toJSON (Search query sFilter sort searchAggs highlight sTrackSortScores sFrom sSize _ sFields sSource) =

22852340 omitNulls [ "query" .= query

@@ -2543,11 +2598,11 @@ instance (FromJSON a) => FromJSON (SearchHits a) where

25432598

25442599instance (FromJSON a) => FromJSON (Hit a) where

25452600 parseJSON (Object v) = Hit <$>

2546− v .: "_index" <*>

2547− v .: "_type" <*>

2548− v .: "_id" <*>

2549− v .: "_score" <*>

2550− v .: "_source" <*>

2601+ v .: "_index" <*>

2602+ v .: "_type" <*>

2603+ v .: "_id" <*>

2604+ v .: "_score" <*>

2605+ v .:? "_source" <*>

25512606 v .:? "highlight"

25522607 parseJSON _ = empty

25532608

stack-7.10.yaml+6−1

@@ -2,7 +2,12 @@ flags: {}

22packages:

33- '.'

44extra-deps:

5+- aeson-0.10.0.0

6+- http-types-0.9

7+- http-client-0.4.24

8+- attoparsec-0.13.0.1

59- doctest-0.10.1

610- doctest-prop-0.2.0.1

711- quickcheck-properties-0.1

8−resolver: lts-3.1

12+- uri-bytestring-0.1.9

13+resolver: lts-3.10

stack-7.8.yamlmode 120000 → 100644+12−1

@@ -1 +0,0 @@

1−stack.yaml

@@ -0,0 +1,12 @@

1+flags: {}

2+packages:

3+- '.'

4+extra-deps:

5+- aeson-0.10.0.0

6+- http-types-0.9

7+- http-client-0.4.24

8+- attoparsec-0.13.0.1

9+- doctest-0.10.1

10+- doctest-prop-0.2.0.1

11+- quickcheck-properties-0.1

12+resolver: lts-2.18

stack.yamlmode 100644 → 120000+1−8

@@ -1,8 +0,0 @@

1−flags: {}

2−packages:

3−- '.'

4−extra-deps:

5−- doctest-0.10.1

6−- doctest-prop-0.2.0.1

7−- quickcheck-properties-0.1

8−resolver: lts-2.18

@@ -0,0 +1 @@

1+stack-7.10.yaml

tests/tests.hs+39−25

@@ -14,6 +14,7 @@ import Data.List (nub)

1414import Data.List.NonEmpty (NonEmpty (..))

1515import qualified Data.List.NonEmpty as NE

1616import qualified Data.Map.Strict as M

17+import qualified Data.Maybe as MY

1718import Data.Monoid

1819import Data.Text (Text)

1920import qualified Data.Text as T

@@ -27,7 +28,7 @@ import Network.HTTP.Client

2728import qualified Network.HTTP.Types.Status as NHTS

2829import Prelude hiding (filter)

2930import Test.Hspec

30−import Test.QuickCheck.Property.Monoid (prop_Monoid, eq, T(..))

31+import Test.QuickCheck.Property.Monoid (T (..), eq, prop_Monoid)

3132

3233import Test.Hspec.QuickCheck (prop)

3334import Test.QuickCheck

@@ -196,7 +197,8 @@ insertWithSpaceInId = do

196197searchTweet :: Search -> BH IO (Either String Tweet)

197198searchTweet search = do

198199 result <- searchTweets search

199− let myTweet = fmap (hitSource . head . hits . searchHits) result

200+ let myTweet :: Either String Tweet

201+ myTweet = grabFirst result

200202 return myTweet

201203

202204searchTweets :: Search -> BH IO (Either String (SearchResult Tweet))

@@ -247,7 +249,7 @@ searchExpectSource src expected = do

247249 let search = (mkSearch (Just query) Nothing) { source = Just src }

248250 reply <- searchAll search

249251 let result = eitherDecode (responseBody reply) :: Either String (SearchResult Value)

250− let value = fmap (hitSource . head . hits . searchHits) result

252+ let value = grabFirst result

251253 liftIO $

252254 value `shouldBe` expected

253255

@@ -313,6 +315,13 @@ instance Arbitrary a => Arbitrary (SearchHits a) where

313315getSource :: EsResult a -> Maybe a

314316getSource = fmap _source . foundResult

315317

318+-- grabFirst :: Either String (SearchResult a) -> Either String a

319+grabFirst r =

320+ case fmap (hitSource . head . hits . searchHits) r of

321+ (Left e) -> Left e

322+ (Right Nothing) -> Left "Source was missing"

323+ (Right (Just x)) -> Right x

324+

316325main :: IO ()

317326main = hspec $ do

318327

@@ -326,6 +335,11 @@ main = hspec $ do

326335 validateStatus resp 200

327336 validateStatus deleteResp 200

328337

338+ describe "error parsing" $ do

339+ it "can parse EsErrors" $ withTestEnv $ do

340+ res <- getDocument (IndexName "bogus") (MappingName "also_bogus") (DocId "bogus_as_well")

341+ let errorResp = eitherDecode (responseBody res)

342+ liftIO (errorResp `shouldBe` Right (EsError 404 "IndexMissingException[[bogus] missing]"))

329343

330344 describe "document API" $ do

331345 it "indexes, gets, and then deletes the generated document" $ withTestEnv $ do

@@ -487,7 +501,7 @@ main = hspec $ do

487501 (Just IdentityFilter) (Just [sortSpec]) Nothing Nothing

488502 False (From 0) (Size 10) SearchTypeQueryThenFetch Nothing Nothing

489503 result <- searchTweets search

490− let myTweet = fmap (hitSource . head . hits . searchHits) result

504+ let myTweet = grabFirst result

491505 liftIO $

492506 myTweet `shouldBe` Right otherTweet

493507

@@ -704,25 +718,25 @@ main = hspec $ do

704718 fmap aggregations res `shouldBe` Right (Just (M.fromList [ docCountPair "bitemyapps" 1

705719 , docCountPair "notmyapps" 1

706720 ]))

707− -- Interaction of date serialization and date histogram aggregation is broken.

708− -- it "returns date histogram aggregation results" $ withTestEnv $ do

709− -- _ <- insertData

710− -- let histogram = DateHistogramAgg $ mkDateHistogram (FieldName "postDate") Minute

711− -- let search = mkAggregateSearch Nothing (mkAggregations "byDate" histogram)

712− -- searchExpectAggs search

713− -- searchValidBucketAgg search "byDate" toDateHistogram

714−

715− -- it "returns date histogram using fractional date" $ withTestEnv $ do

716− -- _ <- insertData

717− -- let periods = [Year, Quarter, Month, Week, Day, Hour, Minute, Second]

718− -- let fractionals = map (FractionalInterval 1.5) [Weeks, Days, Hours, Minutes, Seconds]

719− -- let intervals = periods ++ fractionals

720− -- let histogram = mkDateHistogram (FieldName "postDate")

721− -- let search interval = mkAggregateSearch Nothing $ mkAggregations "byDate" $ DateHistogramAgg (histogram interval)

722− -- let expect interval = searchExpectAggs (search interval)

723− -- let valid interval = searchValidBucketAgg (search interval) "byDate" toDateHistogram

724− -- forM_ intervals expect

725− -- forM_ intervals valid

721+

722+ it "returns date histogram aggregation results" $ withTestEnv $ do

723+ _ <- insertData

724+ let histogram = DateHistogramAgg $ mkDateHistogram (FieldName "postDate") Minute

725+ let search = mkAggregateSearch Nothing (mkAggregations "byDate" histogram)

726+ searchExpectAggs search

727+ searchValidBucketAgg search "byDate" toDateHistogram

728+

729+ it "returns date histogram using fractional date" $ withTestEnv $ do

730+ _ <- insertData

731+ let periods = [Year, Quarter, Month, Week, Day, Hour, Minute, Second]

732+ let fractionals = map (FractionalInterval 1.5) [Weeks, Days, Hours, Minutes, Seconds]

733+ let intervals = periods ++ fractionals

734+ let histogram = mkDateHistogram (FieldName "postDate")

735+ let search interval = mkAggregateSearch Nothing $ mkAggregations "byDate" $ DateHistogramAgg (histogram interval)

736+ let expect interval = searchExpectAggs (search interval)

737+ let valid interval = searchValidBucketAgg (search interval) "byDate" toDateHistogram

738+ forM_ intervals expect

739+ forM_ intervals valid

726740

727741 describe "Highlights API" $ do

728742

@@ -753,7 +767,7 @@ main = hspec $ do

753767 it "doesn't include source when sources are disabled" $ withTestEnv $ do

754768 searchExpectSource

755769 NoSource

756− (Left "key \"_source\" not present")

770+ (Left "Source was missing")

757771

758772 it "includes a source" $ withTestEnv $ do

759773 searchExpectSource

@@ -849,4 +863,4 @@ main = hspec $ do

849863 liftIO $

850864 regular_search `shouldBe` Right exampleTweet -- Check that the size restrtiction is being honored

851865 liftIO $

852− scan_search `shouldMatchList` [exampleTweet, otherTweet]

866+ scan_search `shouldMatchList` [Just exampleTweet, Just otherTweet]