cc93097 · typeable/minio-hs

git.iko.soy/typeable/minio-hs

Fix listing to also return common prefixes (#126)

- Also bump up to 1.4.0 as this is a breaking change.

commitcc930975c9e0a9506eb64506476f5425399586b0 authorAditya Manthramurthy <donatello@users.noreply.github.com> dateTue 9 Jul 2019, 12:20 (7 years ago) parent4a807fd

5 files changed+85−20

compare with master

unifiedsplit

minio-hs.cabal+1−1 src/Network/Minio.hs+2−1 src/Network/Minio/Data.hs+1−1 src/Network/Minio/ListOps.hs+31−11 test/LiveServer.hs+50−6

minio-hs.cabal+1−1

@@ -1,5 +1,5 @@

11name: minio-hs

2−version: 1.3.1

2+version: 1.4.0

33synopsis: A MinIO Haskell Library for Amazon S3 compatible cloud

44 storage.

55description: The MinIO Haskell client library provides simple APIs to

src/Network/Minio.hs+2−1

@@ -1,5 +1,5 @@

11--

33--

44-- Licensed under the Apache License, Version 2.0 (the "License");

55-- you may not use this file except in compliance with the License.

@@ -86,6 +86,7 @@ module Network.Minio

8686 , oiSize

8787 , oiMetadata

8888

89+ , ListItem(..)

8990 , listObjects

9091 , listObjectsV1

9192

src/Network/Minio/Data.hs+1−1

@@ -435,7 +435,7 @@ data ListObjectsV1Result = ListObjectsV1Result {

435435

436436-- | Represents information about an object.

437437data ObjectInfo = ObjectInfo

438− { oiObject :: Object -- ^ Oject key

438+ { oiObject :: Object -- ^ Object key

439439 , oiModTime :: UTCTime -- ^ Mdification time of the object

440440 , oiETag :: ETag -- ^ ETag of the object

441441 , oiSize :: Int64 -- ^ Size of the object in bytes

src/Network/Minio/ListOps.hs+31−11

@@ -1,5 +1,5 @@

11--

33--

44-- Licensed under the Apache License, Version 2.0 (the "License");

55-- you may not use this file except in compliance with the License.

@@ -25,34 +25,54 @@ import Lib.Prelude

2525import Network.Minio.Data

2626import Network.Minio.S3API

2727

28−-- | List objects in a bucket matching the given prefix. If recurse is

29−-- set to True objects matching prefix are recursively listed.

30−listObjects :: Bucket -> Maybe Text -> Bool -> C.ConduitM () ObjectInfo Minio ()

28+-- | Represents a list output item - either an object or an object

29+-- prefix (i.e. a directory).

30+data ListItem = ListItemObject ObjectInfo

31+ | ListItemPrefix Text

32+ deriving (Show, Eq)

33+

34+-- | @'listObjects' bucket prefix recurse@ lists objects in a bucket

35+-- similar to a file system tree traversal.

36+--

37+-- If @prefix@ is not 'Nothing', only items with the given prefix are

38+-- listed, otherwise items under the bucket are returned.

39+--

40+-- If @recurse@ is set to @True@ all directories under the prefix are

41+-- recursively traversed and only objects are returned.

42+--

43+-- If @recurse@ is set to @False@, objects and directories immediately

44+-- under the given prefix are returned (no recursive traversal is

45+-- performed).

46+listObjects :: Bucket -> Maybe Text -> Bool -> C.ConduitM () ListItem Minio ()

3147listObjects bucket prefix recurse = loop Nothing

3248 where

33− loop :: Maybe Text -> C.ConduitM () ObjectInfo Minio ()

49+ loop :: Maybe Text -> C.ConduitM () ListItem Minio ()

3450 loop nextToken = do

3551 let

3652 delimiter = bool (Just "/") Nothing recurse

3753

3854 res <- lift $ listObjects' bucket prefix nextToken delimiter Nothing

39− CL.sourceList $ lorObjects res

55+ CL.sourceList $ map ListItemObject $ lorObjects res

56+ unless recurse $

57+ CL.sourceList $ map ListItemPrefix $ lorCPrefixes res

4058 when (lorHasMore res) $

4159 loop (lorNextToken res)

4260

43−-- | List objects in a bucket matching the given prefix. If recurse is

44−-- set to True objects matching prefix are recursively listed.

61+-- | Lists objects - similar to @listObjects@, however uses the older

62+-- V1 AWS S3 API. Prefer @listObjects@ to this.

4563listObjectsV1 :: Bucket -> Maybe Text -> Bool

46− -> C.ConduitM () ObjectInfo Minio ()

64+ -> C.ConduitM () ListItem Minio ()

4765listObjectsV1 bucket prefix recurse = loop Nothing

4866 where

49− loop :: Maybe Text -> C.ConduitM () ObjectInfo Minio ()

67+ loop :: Maybe Text -> C.ConduitM () ListItem Minio ()

5068 loop nextMarker = do

5169 let

5270 delimiter = bool (Just "/") Nothing recurse

5371

5472 res <- lift $ listObjectsV1' bucket prefix nextMarker delimiter Nothing

55− CL.sourceList $ lorObjects' res

73+ CL.sourceList $ map ListItemObject $ lorObjects' res

74+ unless recurse $

75+ CL.sourceList $ map ListItemPrefix $ lorCPrefixes' res

5676 when (lorHasMore' res) $

5777 loop (lorNextMarker res)

5878

test/LiveServer.hs+50−6

@@ -1,6 +1,6 @@

11{-# LANGUAGE OverloadedStrings #-}

22--

44--

55-- Licensed under the Apache License, Version 2.0 (the "License");

66-- you may not use this file except in compliance with the License.

@@ -295,22 +295,66 @@ highLevelListingTest = funTestWithBucket "High-level listObjects Test" $

295295 \step bucket -> do

296296 step "High-level listObjects Test"

297297 step "put 3 objects"

298− let expectedObjects = ["dir/o1", "dir/dir1/o2", "dir/dir2/o3"]

298+ let expectedObjects = ["dir/o1", "dir/dir1/o2", "dir/dir2/o3", "o4"]

299+ extractObjectsFromList os =

300+ mapM (\t -> case t of

301+ ListItemObject o -> Just $ oiObject o

302+ _ -> Nothing) os

303+ expectedNonRecList = ["o4", "dir/"]

304+ extractObjectsAndDirsFromList os =

305+ map (\t -> case t of

306+ ListItemObject o -> oiObject o

307+ ListItemPrefix d -> d) os

308+

299309 forM_ expectedObjects $

300310 \obj -> fPutObject bucket obj "/etc/lsb-release" defaultPutObjectOptions

301311

302312 step "High-level listing of objects"

313+ items <- C.runConduit $ listObjects bucket Nothing False C..| sinkList

314+ liftIO $ assertEqual "Objects/Dirs match failed!" expectedNonRecList $

315+ extractObjectsAndDirsFromList items

316+

317+ step "High-level recursive listing of objects"

303318 objects <- C.runConduit $ listObjects bucket Nothing True C..| sinkList

304319

305− liftIO $ assertEqual "Objects match failed!" (sort expectedObjects)

306− (map oiObject objects)

320+ liftIO $ assertEqual "Objects match failed!"

321+ (Just $ sort expectedObjects) $

322+ extractObjectsFromList objects

307323

308324 step "High-level listing of objects (version 1)"

325+ itemsV1 <- C.runConduit $ listObjectsV1 bucket Nothing False C..| sinkList

326+ liftIO $ assertEqual "Objects/Dirs match failed!" expectedNonRecList $

327+ extractObjectsAndDirsFromList itemsV1

328+

329+ step "High-level recursive listing of objects (version 1)"

309330 objectsV1 <- C.runConduit $ listObjectsV1 bucket Nothing True C..|

310331 sinkList

311332

312− liftIO $ assertEqual "Objects match failed!" (sort expectedObjects)

313− (map oiObject objectsV1)

333+ liftIO $ assertEqual "Objects match failed!"

334+ (Just $ sort expectedObjects) $

335+ extractObjectsFromList objectsV1

336+

337+ let expectedPrefListing = ["dir/o1", "dir/dir1/", "dir/dir2/"]

338+ expectedPrefListingRec = Just ["dir/dir1/o2", "dir/dir2/o3", "dir/o1"]

339+ step "High-level listing with prefix"

340+ prefItems <- C.runConduit $ listObjects bucket (Just "dir/") False C..| sinkList

341+ liftIO $ assertEqual "Objects/Dirs under prefix match failed!"

342+ expectedPrefListing $ extractObjectsAndDirsFromList prefItems

343+

344+ step "High-level listing with prefix recursive"

345+ prefItemsRec <- C.runConduit $ listObjects bucket (Just "dir/") True C..| sinkList

346+ liftIO $ assertEqual "Objects/Dirs under prefix match recursive failed!"

347+ expectedPrefListingRec $ extractObjectsFromList prefItemsRec

348+

349+ step "High-level listing with prefix (version 1)"

350+ prefItemsV1 <- C.runConduit $ listObjectsV1 bucket (Just "dir/") False C..| sinkList

351+ liftIO $ assertEqual "Objects/Dirs under prefix match failed!"

352+ expectedPrefListing $ extractObjectsAndDirsFromList prefItemsV1

353+

354+ step "High-level listing with prefix recursive (version 1)"

355+ prefItemsRecV1 <- C.runConduit $ listObjectsV1 bucket (Just "dir/") True C..| sinkList

356+ liftIO $ assertEqual "Objects/Dirs under prefix match recursive failed!"

357+ expectedPrefListingRec $ extractObjectsFromList prefItemsRecV1

314358

315359 step "Cleanup actions"

316360 forM_ expectedObjects $