3236d73 · streamly · git.iko.soy

git.iko.soy/streamly

on master help sign in

Add fromPureStream and fromByteStr# to Arrays

3236d73Ranjeet Kumar Ranjan3 years ago

core/src/Streamly/Internal/Data/Array/Generic.hs+14

compare with master

unifiedsplit

@@ -18,6 +18,8 @@ module Streamly.Internal.Data.Array.Generic

19 , fromStreamN

20 , fromStream

21+ , fromPureStream

22+ , fromByteStr#

22 , fromListN

24 , fromListN

23 , fromList

25 , fromList

@@ -47,7 +49,10 @@ where

48import Control.Monad (replicateM)

50import Control.Monad (replicateM)

49import Control.Monad.IO.Class (MonadIO)

51import Control.Monad.IO.Class (MonadIO)

52+import Data.Functor.Identity (Identity(..))

53+import Data.Word (Word8)

50import GHC.Base (MutableArray#, RealWorld)

54import GHC.Base (MutableArray#, RealWorld)

55+import GHC.Exts (Addr#)

51import GHC.IO (unsafePerformIO)

56import GHC.IO (unsafePerformIO)

52import Text.Read (readPrec)

57import Text.Read (readPrec)

@@ -113,6 +118,15 @@ writeWith elemCount = unsafeFreeze <$> MArray.writeWith elemCount

113write :: MonadIO m => Fold m a (Array a)

118write :: MonadIO m => Fold m a (Array a)

114write = fmap unsafeFreeze MArray.write

119write = fmap unsafeFreeze MArray.write

115

120

121+fromPureStream :: Stream Identity a -> Array a

122+fromPureStream x =

123+ unsafePerformIO $ fmap (unsafeFreeze) (MArray.fromPureStream x)

124+-- fromPureStream = runIdentity . D.fold (unsafeMakePure write)

125+-- fromPureStream = fromList . runIdentity . D.toList

126+

127+fromByteStr# :: Addr# -> Array Word8

128+fromByteStr# addr = fromPureStream (D.fromByteStr# addr)

129+

116-------------------------------------------------------------------------------

130-------------------------------------------------------------------------------

117-- Construction - from streams

131-- Construction - from streams

118-------------------------------------------------------------------------------

132-------------------------------------------------------------------------------

core/src/Streamly/Internal/Data/Array/Generic/Mut/Type.hs+8

@@ -29,6 +29,7 @@ module Streamly.Internal.Data.Array.Generic.Mut.Type

29 , write

30 , fromStreamN

31 , fromStream

32+ , fromPureStream

33 -- , writeRevN

34 -- , writeRevN

34 -- , writeRev

35 -- , writeRev

@@ -166,6 +167,7 @@ where

166

167

167import Control.Monad (when)

168import Control.Monad (when)

168import Control.Monad.IO.Class (MonadIO(..))

169import Control.Monad.IO.Class (MonadIO(..))

170+import Data.Functor.Identity (Identity(..))

169import GHC.Base

171import GHC.Base

170 ( MutableArray#

172 ( MutableArray#

171 , RealWorld

173 , RealWorld

@@ -185,6 +187,7 @@ import qualified Streamly.Internal.Data.Fold.Type as FL

185import qualified Streamly.Internal.Data.Producer as Producer

187import qualified Streamly.Internal.Data.Producer as Producer

186import qualified Streamly.Internal.Data.Stream.StreamD.Type as D

188import qualified Streamly.Internal.Data.Stream.StreamD.Type as D

187import qualified Streamly.Internal.Data.Stream.StreamD.Generate as D

189import qualified Streamly.Internal.Data.Stream.StreamD.Generate as D

190+import qualified Streamly.Internal.Data.Stream.StreamD.Lift as D

188import qualified Streamly.Internal.Data.Stream.StreamK.Type as K

191import qualified Streamly.Internal.Data.Stream.StreamK.Type as K

189

192

190import Prelude hiding (read, length)

193import Prelude hiding (read, length)

@@ -658,6 +661,11 @@ fromListN n xs = fromStreamN n $ D.fromList xs

658fromList :: MonadIO m => [a] -> m (MutArray a)

661fromList :: MonadIO m => [a] -> m (MutArray a)

659fromList xs = fromStream $ D.fromList xs

662fromList xs = fromStream $ D.fromList xs

660

663

664+{-# INLINABLE fromPureStream #-}

665+fromPureStream :: MonadIO m => Stream Identity a -> m (MutArray a)

666+fromPureStream xs =

667+ liftIO $ D.fold write $ D.morphInner (return . runIdentity) xs

668+

661-------------------------------------------------------------------------------

669-------------------------------------------------------------------------------

662-- Unfolds

670-- Unfolds

663-------------------------------------------------------------------------------

671-------------------------------------------------------------------------------

core/src/Streamly/Internal/Data/Array/Mut/Type.hs+9

@@ -71,6 +71,7 @@ module Streamly.Internal.Data.Array.Mut.Type

71 , fromListRev

72 , fromStreamDN

73 , fromStreamD

74+ , fromPureStream

75 -- * Random writes

76 -- * Random writes

76 , putIndex

77 , putIndex

@@ -227,6 +228,7 @@ where

227import Control.Monad (when, void)

228import Control.Monad (when, void)

228import Control.Monad.IO.Class (MonadIO(..))

229import Control.Monad.IO.Class (MonadIO(..))

229import Data.Bits (shiftR, (.|.), (.&.))

230import Data.Bits (shiftR, (.|.), (.&.))

231+import Data.Functor.Identity (Identity(..))

230import Data.Proxy (Proxy(..))

232import Data.Proxy (Proxy(..))

231import Data.Word (Word8)

233import Data.Word (Word8)

232import Foreign.C.Types (CSize(..), CInt(..))

234import Foreign.C.Types (CSize(..), CInt(..))

@@ -262,6 +264,7 @@ import Streamly.Internal.System.IO (arrayPayloadSize, defaultChunkSize)

262import qualified Streamly.Internal.Data.Fold.Type as FL

264import qualified Streamly.Internal.Data.Fold.Type as FL

263import qualified Streamly.Internal.Data.Producer as Producer

265import qualified Streamly.Internal.Data.Producer as Producer

264import qualified Streamly.Internal.Data.Stream.StreamD.Type as D

266import qualified Streamly.Internal.Data.Stream.StreamD.Type as D

267+import qualified Streamly.Internal.Data.Stream.StreamD.Lift as D

265import qualified Streamly.Internal.Data.Stream.StreamK.Type as K

268import qualified Streamly.Internal.Data.Stream.StreamK.Type as K

266import qualified Streamly.Internal.Data.Unbox as Unboxed

269import qualified Streamly.Internal.Data.Unbox as Unboxed

267import qualified Prelude

270import qualified Prelude

@@ -1949,6 +1952,12 @@ fromListN n xs = fromStreamDN n $ D.fromList xs

1949fromListRevN :: (MonadIO m, Unbox a) => Int -> [a] -> m (MutArray a)

1952fromListRevN :: (MonadIO m, Unbox a) => Int -> [a] -> m (MutArray a)

1950fromListRevN n xs = D.fold (writeRevN n) $ D.fromList xs

1953fromListRevN n xs = D.fold (writeRevN n) $ D.fromList xs

1951

1954

1955+-- | Convert a pure stream in Identity monad to a mutable array.

1956+{-# INLINABLE fromPureStream #-}

1957+fromPureStream :: (MonadIO m, Unbox a) => Stream Identity a -> m (MutArray a)

1958+fromPureStream xs =

1959+ liftIO $ D.fold write $ D.morphInner (return . runIdentity) xs

1960+

1952-------------------------------------------------------------------------------

1961-------------------------------------------------------------------------------

1953-- convert stream to a single array

1962-- convert stream to a single array

1954-------------------------------------------------------------------------------

1963-------------------------------------------------------------------------------

core/src/Streamly/Internal/Data/Array/Type.hs+61−1

@@ -34,6 +34,8 @@ module Streamly.Internal.Data.Array.Type

34 , fromListRevN

35 , fromStreamDN

36 , fromStreamD

37+ , fromPureStream

38+ , fromByteStr#

38 -- * Split

40 -- * Split

39 , breakOn

41 , breakOn

@@ -66,6 +68,7 @@ module Streamly.Internal.Data.Array.Type

66 , MA.ArrayUnsafe (..)

68 , MA.ArrayUnsafe (..)

67 , writeNAligned

69 , writeNAligned

68 , write

70 , write

71+ , unsafeMakePure

70 -- * Streams of arrays

73 -- * Streams of arrays

71 , chunksOf

74 , chunksOf

@@ -85,7 +88,7 @@ import Data.Functor.Identity (Identity(..))

85import Data.Proxy (Proxy(..))

88import Data.Proxy (Proxy(..))

86import Data.Word (Word8)

89import Data.Word (Word8)

87import GHC.Base (build)

90import GHC.Base (build)

88−import GHC.Exts (IsList, IsString(..))

91+import GHC.Exts (IsList, IsString(..), Addr#)

90import GHC.IO (unsafePerformIO)

93import GHC.IO (unsafePerformIO)

91import GHC.Ptr (Ptr(..))

94import GHC.Ptr (Ptr(..))

@@ -101,6 +104,7 @@ import Prelude hiding (Foldable(..), read, unlines, splitAt)

101import qualified GHC.Exts as Exts

104import qualified GHC.Exts as Exts

102import qualified Streamly.Internal.Data.Array.Mut.Type as MA

105import qualified Streamly.Internal.Data.Array.Mut.Type as MA

103import qualified Streamly.Internal.Data.Stream.StreamD.Type as D

106import qualified Streamly.Internal.Data.Stream.StreamD.Type as D

107+import qualified Streamly.Internal.Data.Stream.StreamD.Generate as D

104import qualified Streamly.Internal.Data.Stream.StreamK.Type as K

108import qualified Streamly.Internal.Data.Stream.StreamK.Type as K

105import qualified Streamly.Internal.Data.Unbox as Unboxed

109import qualified Streamly.Internal.Data.Unbox as Unboxed

106import qualified Streamly.Internal.Data.Unfold.Type as Unfold

110import qualified Streamly.Internal.Data.Unfold.Type as Unfold

@@ -495,6 +499,62 @@ writeWith elemCount = unsafeFreeze <$> MA.writeWith elemCount

495write :: forall m a. (MonadIO m, Unbox a) => Fold m a (Array a)

499write :: forall m a. (MonadIO m, Unbox a) => Fold m a (Array a)

496write = fmap unsafeFreeze MA.write

500write = fmap unsafeFreeze MA.write

497

501

502+-- | Fold "step" has a dependency on "initial", and each step is dependent on

503+-- the previous invocation of step due to state passing, finally extract

504+-- depends on the result of step, therefore, as long as the fold is driven in

505+-- the correct order the operations would be correctly ordered. We need to

506+-- ensure that we strictly evaluate the previous step completely before the

507+-- next step.

508+--

509+-- To not share the same array we need to make sure that the result of

510+-- "initial" is not shared. Existential type ensures that it does not get

511+-- shared across different folds. However, if we invoke "initial" multiple

512+-- times for the same fold, there is a possiblity of sharing the two because

513+-- the compiler would consider it as a pure value. One such example is the

514+-- chunksOf combinator, or using an array creation fold with foldMany

515+-- combinator. Is there a proper way in GHC to tell it to not share a pure

516+-- expression in a particular case?

517+--

518+-- For this reason array creation folds have a MonadIO constraint. Pure folds

519+-- could be unsafe and dangerous. This is dangerous especially when used with

520+-- foldMany like operations.

521+--

522+-- >>> import qualified Streamly.Internal.Data.Array.Type as Array

523+-- >>> unsafePureWrite = Array.unsafeMakePure Array.write

524+--

525+{-# INLINE unsafeMakePure #-}

526+unsafeMakePure :: Monad m => Fold IO a b -> Fold m a b

527+unsafeMakePure (Fold step initial extract) =

528+ Fold (\x a -> return $! unsafeInlineIO (step x a))

529+ (return $! unsafePerformIO initial)

530+ (\s -> return $! unsafeInlineIO $ extract s)

531+

532+-- | Convert a pure stream in Identity monad to an immutable array.

533+--

534+-- Same as the following but with better performance:

535+--

536+-- >>> fromPureStream = Array.fromList . runIdentity . Stream.toList

537+--

538+fromPureStream :: Unbox a => Stream Identity a -> Array a

539+fromPureStream x = unsafePerformIO $ fmap (unsafeFreeze) (MA.fromPureStream x)

540+-- fromPureStream = runIdentity . D.fold (unsafeMakePure write)

541+-- fromPureStream = fromList . runIdentity . D.toList

542+

543+-- | Copy a null terminated immutable 'Addr#' Word8 sequence into an array.

544+--

545+-- /Unsafe:/ The caller is responsible for safe addressing.

546+--

547+-- Note that this is completely safe when reading from Haskell string

548+-- literals because they are guaranteed to be NULL terminated:

549+--

550+-- >>> :set -XMagicHash

551+-- >>> import qualified Streamly.Internal.Data.Array.Type as Array

552+-- >>> Array.toList $ Array.fromByteStr# "\1\2\3\0"#

553+-- [1,2,3]

554+--

555+fromByteStr# :: Addr# -> Array Word8

556+fromByteStr# addr = fromPureStream (D.fromByteStr# addr)

557+

498-------------------------------------------------------------------------------

558-------------------------------------------------------------------------------

499-- Instances

559-- Instances

500-------------------------------------------------------------------------------

560-------------------------------------------------------------------------------