abdc6e1 · streamly · git.iko.soy

core/docs/Changelog.md+5−2

compare with master

unifiedsplit

@@ -2,8 +2,11 @@

2

3## Unreleased

4

5−* Array creation APIs now return unpinned arrays. To create pinned arrays, use

5+* Arrays are now created unpinned by default, they were created pinned

6+ earlier. During IO operations unpinned arrays are automatically copied

7+ to pinned memory. When arrays are directly passed to IO operations

8+ programmers can choose to create them pinned to avoid a copy. To

6− the internal APIs with the `pinned*` prefix.

9+ create pinned arrays, use the internal APIs with the `pinned*` prefix.

7

10

8### Deprecations

11### Deprecations

9

12

core/src/Streamly/Data/Array.hs+10−14

@@ -117,9 +117,11 @@ import Prelude hiding (read)

117--

118-- == Pinned and Unpinned Arrays

119--

120−-- The array type can use both pinned and unpinned memory under the hood. The

120+-- The array type can use both pinned and unpinned memory under the hood. The

121−-- arrays are unpinned by default and are only pinned while doing any I/O

121+-- default array creation operations create unpinned arrays. IO operations

122−-- operations.

122+-- automatically copy an array to pinned memory if the array passed to it is

123+-- unpinned. Programmers can use appropriate pinned array generation APIs to

124+-- reduce the copying if it happens.

123--

125--

124-- Unpinned arrays have the advantage of allowing automatic defragmentation of

126-- Unpinned arrays have the advantage of allowing automatic defragmentation of

125-- the memory by GC. Whereas pinned arrays have the advantage of not requiring

127-- the memory by GC. Whereas pinned arrays have the advantage of not requiring

@@ -130,20 +132,14 @@ import Prelude hiding (read)

130--

132--

131-- == Creating Arrays from Non-IO Streams

133-- == Creating Arrays from Non-IO Streams

132--

134--

133−-- Array creation folds require 'MonadIO' because they need to sequence effects

135+-- Array creation folds require 'MonadIO' otherwise the compiler may

134−-- in IO streams. To operate on streams in pure Monads like 'Identity' you can

136+-- incorrectly share the array memory thinking it is pure.

135−-- morph it to IO monad as follows:

136--

137--

137−-- The 'MonadIO' based folds can be morphed to 'Identity' stream folds:

138+-- See the 'fromPureStream' unreleased API to generate an array from an

139+-- Identity stream safely without using MonadIO constraint.

138--

140--

139−-- >>> purely = Fold.morphInner (Identity . unsafePerformIO)

140−-- >>> Stream.fold (purely Array.write) $ Stream.fromList [1,2,3::Int]

141−-- Identity fromList [1,2,3]

142--

141--

143−-- Since it is a pure stream we can use 'unsafePerformIO' to extract the result

144−-- of fold from IO.

145−--

146−-- Alternatively, 'Identity' streams can be generalized to IO streams:

142+-- Note that 'Identity' streams can be generalized to IO streams:

147--

143--

148-- >>> pure = Stream.fromList [1,2,3] :: Stream Identity Int

144-- >>> pure = Stream.fromList [1,2,3] :: Stream Identity Int

149-- >>> generally = Stream.morphInner (return . runIdentity)

145-- >>> generally = Stream.morphInner (return . runIdentity)

core/src/Streamly/Internal/Data/Array/Mut/Type.hs+10−2

@@ -773,6 +773,7 @@ reallocExplicit elemSize newCapacityInBytes MutArray{..} = do

773-- down to multiples of array size. If the new size is more than

774-- 'largeObjectThreshold' then it is rounded up to the block size (4K).

775--

776+-- If the original array is pinned, the newly allocated array is also pinned.

776{-# INLINABLE realloc #-}

777{-# INLINABLE realloc #-}

777realloc :: forall m a. (MonadIO m, Unbox a) => Int -> MutArray a -> m (MutArray a)

778realloc :: forall m a. (MonadIO m, Unbox a) => Int -> MutArray a -> m (MutArray a)

778realloc bytes arr = liftIO $ reallocExplicit (SIZE_OF(a)) bytes arr

779realloc bytes arr = liftIO $ reallocExplicit (SIZE_OF(a)) bytes arr

@@ -2094,7 +2095,8 @@ putSliceUnsafe src srcStartBytes dst dstStartBytes lenBytes = liftIO $ do

2094 arrS# srcStartBytes# arrD# dstStartBytes# lenBytes# s#

2095 arrS# srcStartBytes# arrD# dstStartBytes# lenBytes# s#

2095 , () #)

2096 , () #)

2096

2097

2097−-- | Copy two arrays into a newly allocated array.

2098+-- | Copy two arrays into a newly allocated array. If the first array is pinned

2099+-- the spliced array is also pinned.

2098{-# INLINE spliceCopy #-}

2100{-# INLINE spliceCopy #-}

2099spliceCopy :: forall m a. MonadIO m =>

2101spliceCopy :: forall m a. MonadIO m =>

2100#ifdef DEVBUILD

2102#ifdef DEVBUILD

@@ -2106,7 +2108,11 @@ spliceCopy arr1 arr2 = liftIO $ do

2106 start2 = arrStart arr2

2108 start2 = arrStart arr2

2107 len1 = arrEnd arr1 - start1

2109 len1 = arrEnd arr1 - start1

2108 len2 = arrEnd arr2 - start2

2110 len2 = arrEnd arr2 - start2

2111+ let newLen = len1 + len2

2109− newArrContents <- liftIO $ Unboxed.newUnpinnedBytes (len1 + len2)

2112+ newArrContents <-

2113+ if Unboxed.isPinned (arrContents arr1)

2114+ then Unboxed.newPinnedBytes newLen

2115+ else Unboxed.newUnpinnedBytes newLen

2110 let len = len1 + len2

2116 let len = len1 + len2

2111 newArr = MutArray newArrContents 0 len len

2117 newArr = MutArray newArrContents 0 len len

2112 putSliceUnsafe arr1 start1 newArr 0 len1

2118 putSliceUnsafe arr1 start1 newArr 0 len1

@@ -2169,6 +2175,8 @@ spliceWith sizer dst@(MutArray _ start end bound) src = do

2169--

2175--

2170-- >>> splice = MutArray.spliceWith (+)

2176-- >>> splice = MutArray.spliceWith (+)

2171--

2177--

2178+-- If the original array is pinned the spliced array is also pinned.

2179+--

2172-- /Pre-release/

2180-- /Pre-release/

2173{-# INLINE splice #-}

2181{-# INLINE splice #-}

2174splice :: (MonadIO m, Unbox a) => MutArray a -> MutArray a -> m (MutArray a)

2182splice :: (MonadIO m, Unbox a) => MutArray a -> MutArray a -> m (MutArray a)

core/src/Streamly/Internal/Data/Unbox.hs+18−2

@@ -53,6 +53,9 @@ import Data.Functor.Const (Const(..))

53import Data.Functor.Identity (Identity(..))

54import Data.Kind (Type)

55import Data.Proxy (Proxy (..))

56+#ifdef DEBUG

57+import Debug.Trace (trace)

58+#endif

56import Foreign.Ptr (IntPtr(..), WordPtr(..))

59import Foreign.Ptr (IntPtr(..), WordPtr(..))

57import GHC.Base (IO(..))

60import GHC.Base (IO(..))

58import GHC.Fingerprint.Type (Fingerprint(..))

61import GHC.Fingerprint.Type (Fingerprint(..))

@@ -150,6 +153,7 @@ newBytes Pinned = pinnedNewBytes

150isPinned :: MutableByteArray -> Bool

153isPinned :: MutableByteArray -> Bool

151isPinned (MutableByteArray arr#) =

154isPinned (MutableByteArray arr#) =

152 let pinnedInt = I# (isMutableByteArrayPinned# arr#)

155 let pinnedInt = I# (isMutableByteArrayPinned# arr#)

156+ -- XXX /= 0 would be safer

153 in pinnedInt == 1

157 in pinnedInt == 1

154

158

155

159

@@ -173,7 +177,13 @@ pin :: MutableByteArray -> IO MutableByteArray

173pin arr@(MutableByteArray marr#) =

177pin arr@(MutableByteArray marr#) =

174 if isPinned arr

178 if isPinned arr

175 then return arr

179 then return arr

176− else IO

180+ else

181+#ifdef DEBUG

182+ do

183+ -- XXX dump stack trace

184+ trace ("pin: Copying array") (return ())

185+#endif

186+ IO

177 $ \s# ->

187 $ \s# ->

178 case cloneMutableArrayWith# newPinnedByteArray# marr# s# of

188 case cloneMutableArrayWith# newPinnedByteArray# marr# s# of

179 (# s1#, marr1# #) -> (# s1#, MutableByteArray marr1# #)

189 (# s1#, marr1# #) -> (# s1#, MutableByteArray marr1# #)

@@ -183,7 +193,13 @@ unpin :: MutableByteArray -> IO MutableByteArray

183unpin arr@(MutableByteArray marr#) =

193unpin arr@(MutableByteArray marr#) =

184 if not (isPinned arr)

194 if not (isPinned arr)

185 then return arr

195 then return arr

186− else IO

196+ else

197+#ifdef DEBUG

198+ do

199+ -- XXX dump stack trace

200+ trace ("unpin: Copying array") (return ())

201+#endif

202+ IO

187 $ \s# ->

203 $ \s# ->

188 case cloneMutableArrayWith# newByteArray# marr# s# of

204 case cloneMutableArrayWith# newByteArray# marr# s# of

189 (# s1#, marr1# #) -> (# s1#, MutableByteArray marr1# #)

205 (# s1#, marr1# #) -> (# s1#, MutableByteArray marr1# #)

core/src/Streamly/Internal/FileSystem/Dir.hs+2−2

@@ -444,7 +444,7 @@ fromChunksWithBufferOf n h xs = fromChunks h $ AS.compact n xs

444-- @since 0.7.0

445{-# INLINE fromStreamWithBufferOf #-}

446fromStreamWithBufferOf :: MonadIO m => Int -> Handle -> Stream m Word8 -> m ()

447−fromStreamWithBufferOf n h m = fromChunks h $ S.chunksOf n m

447+fromStreamWithBufferOf n h m = fromChunks h $ S.chunksOfWith Pinned n m

448-- fromStreamWithBufferOf n h m = fromChunks h $ AS.chunksOf n m

449

450-- > write = 'writeWithBufferOf' A.defaultChunkSize

@@ -495,7 +495,7 @@ writeChunksWithBufferOf n h = lpackArraysChunksOf n (writeChunks h)

495-- @since 0.7.0

496{-# INLINE writeWithBufferOf #-}

497writeWithBufferOf :: MonadIO m => Int -> Handle -> Fold m Word8 ()

498−writeWithBufferOf n h = FL.groupsOf n (writeNUnsafe n) (writeChunks h)

498+writeWithBufferOf n h = FL.groupsOf n (writeNUnsafeUsing Pinned n) (writeChunks h)

499

500-- > write = 'writeWithBufferOf' A.defaultChunkSize

501--

core/src/Streamly/Internal/FileSystem/Handle.hs+2−2

@@ -425,12 +425,12 @@ putChunksWith :: (MonadIO m, Unbox a)

425 => Int -> Handle -> Stream m (Array a) -> m ()

426putChunksWith n h xs = putChunks h $ AS.compact n xs

427

428+-- > putBytesWith n h m = Handle.putChunks h $ A.chunksOfWith Pinned n m

429+

428-- | @putBytesWith bufsize handle stream@ writes @stream@ to @handle@

430-- | @putBytesWith bufsize handle stream@ writes @stream@ to @handle@

429-- in chunks of @bufsize@. A write is performed to the IO device as soon as we

431-- in chunks of @bufsize@. A write is performed to the IO device as soon as we

430-- collect the required input size.

432-- collect the required input size.

431--

433--

432−-- >>> putBytesWith n h m = Handle.putChunks h $ Stream.chunksOf n m

433−--

434{-# INLINE putBytesWith #-}

435putBytesWith :: MonadIO m => Int -> Handle -> Stream m Word8 -> m ()

436putBytesWith n h m = putChunks h $ A.chunksOfAs Pinned n m