53a6a87 · streamly · git.iko.soy

src/Streamly/Internal.hs+1

compare with master

unifiedsplit

@@ -46,6 +46,7 @@ module Streamly.Internal

46 , splitOnSuffixSeq

47 , splitBySeq

48 , splitWithSuffixSeq

49+ , splitInnerBy

49

50

50 , tapAsync

51 , tapAsync

51

52

src/Streamly/Memory/Array/Types.hs+7−4

@@ -28,6 +28,7 @@ module Streamly.Memory.Array.Types

28 , unsafeSnoc

29 , snoc

30 , spliceWithDoubling

31+ , spliceTwo

31

32

32 , fromList

33 , fromList

33 , fromListN

34 , fromListN

@@ -45,6 +46,7 @@ module Streamly.Memory.Array.Types

45 , groupIOVecsOf

46 , groupIOVecsOf

46#endif

47#endif

47 , splitOn

48 , splitOn

49+ , breakOn

48

50

49 -- * Elimination

51 -- * Elimination

50 , unsafeIndexIO

52 , unsafeIndexIO

@@ -942,6 +944,7 @@ unlines sep (D.Stream step state) = D.Stream step' (OuterLoop state)

942 return $ D.Yield x (InnerLoop st startf

944 return $ D.Yield x (InnerLoop st startf

943 (p `plusPtr` (sizeOf (undefined :: a))) end)

945 (p `plusPtr` (sizeOf (undefined :: a))) end)

944

946

947+-- Splice two immutable arrays creating a new array.

945{-# INLINE spliceTwo #-}

948{-# INLINE spliceTwo #-}

946spliceTwo :: (MonadIO m, Storable a) => Array a -> Array a -> m (Array a)

949spliceTwo :: (MonadIO m, Storable a) => Array a -> Array a -> m (Array a)

947spliceTwo arr1 arr2 = do

950spliceTwo arr1 arr2 = do

@@ -960,8 +963,8 @@ spliceTwo arr1 arr2 = do

960 touchForeignPtr (aStart arr2)

963 touchForeignPtr (aStart arr2)

961 return arr { aEnd = dst `plusPtr` (len1 + len2) }

964 return arr { aEnd = dst `plusPtr` (len1 + len2) }

962

965

963−-- Splice a new array into a pre-reserved array. The user must ensure that

966+-- Splice an array into a pre-reserved mutable array. The user must ensure

964−-- there is enough space in the array.

967+-- that there is enough space in the mutable array.

965{-# INLINE spliceWith #-}

968{-# INLINE spliceWith #-}

966spliceWith :: (MonadIO m) => Array a -> Array a -> m (Array a)

969spliceWith :: (MonadIO m) => Array a -> Array a -> m (Array a)

967spliceWith dst@(Array _ end bound) src = liftIO $ do

970spliceWith dst@(Array _ end bound) src = liftIO $ do

@@ -975,8 +978,8 @@ spliceWith dst@(Array _ end bound) src = liftIO $ do

975 memcpy (castPtr pdst) (castPtr psrc) srcLen

978 memcpy (castPtr pdst) (castPtr psrc) srcLen

976 return $ dst { aEnd = pdst `plusPtr` srcLen }

979 return $ dst { aEnd = pdst `plusPtr` srcLen }

977

980

978−-- Splice a new array into a preallocated array, doubling the space if there is

981+-- Splice a new array into a preallocated mutable array, doubling the space if

979−-- no space in the target array.

982+-- there is no space in the target array.

980{-# INLINE spliceWithDoubling #-}

983{-# INLINE spliceWithDoubling #-}

981spliceWithDoubling :: (MonadIO m, Storable a)

984spliceWithDoubling :: (MonadIO m, Storable a)

982 => Array a -> Array a -> m (Array a)

985 => Array a -> Array a -> m (Array a)

src/Streamly/Memory/ArrayStream.hs+3−1

@@ -109,7 +109,9 @@ splitOn

109 => Word8

110 -> t m (Array Word8)

111 -> t m (Array Word8)

112−splitOn byte s = D.fromStreamD $ A.splitOn byte $ D.toStreamD s

112+-- splitOn byte s = D.fromStreamD $ A.splitOn byte $ D.toStreamD s

113+splitOn byte s =

114+ D.fromStreamD $ D.splitInnerBy (A.breakOn byte) A.spliceTwo $ D.toStreamD s

113

115

114-- | Coalesce adjacent arrays in incoming stream to form bigger arrays of a

116-- | Coalesce adjacent arrays in incoming stream to form bigger arrays of a

115-- maximum specified size in bytes.

117-- maximum specified size in bytes.

src/Streamly/Prelude/Internal.hs+28

@@ -300,6 +300,9 @@ module Streamly.Prelude.Internal

300 -- , splitOnAnySuffixSeq

301 -- , splitOnAnyPrefixSeq

302

303+ -- Nested splitting

304+ , splitInnerBy

305+

303 -- ** Grouping

306 -- ** Grouping

304 , groups

307 , groups

305 , groupsBy

308 , groupsBy

@@ -3012,6 +3015,31 @@ splitSuffixOnAny subseq f m = undefined

3012 -- D.fromStreamD $ D.splitPostAny f subseq (D.toStreamD m)

3015 -- D.fromStreamD $ D.splitPostAny f subseq (D.toStreamD m)

3013-}

3016-}

3014

3017

3018+------------------------------------------------------------------------------

3019+-- Nested Split

3020+------------------------------------------------------------------------------

3021+

3022+-- | Consider a chunked stream of container elements e.g. a stream of @Word8@

3023+-- chunked as a stream of arrays of @Word8@. @splitInnerBy splitter joiner

3024+-- stream@ splits the inner containers @f a@ using the @splitter@ function and

3025+-- joins back the resulting fragments from splitting across multiple containers

3026+-- using the @joiner@ function such that the transformed output stream is

3027+-- consolidated as one container per segment of the split.

3028+--

3029+-- CAUTION! This is not a true streaming function as the container size after

3030+-- the split and merge may not be bounded.

3031+--

3032+-- @since 0.7.0

3033+{-# INLINE splitInnerBy #-}

3034+splitInnerBy

3035+ :: (IsStream t, Monad m)

3036+ => (f a -> m (f a, Maybe (f a))) -- splitter

3037+ -> (f a -> f a -> m (f a)) -- joiner

3038+ -> t m (f a)

3039+ -> t m (f a)

3040+splitInnerBy splitter joiner xs =

3041+ D.fromStreamD $ D.splitInnerBy splitter joiner $ D.toStreamD xs

3042+

3015------------------------------------------------------------------------------

3043------------------------------------------------------------------------------

3016-- Reorder in sequence

3044-- Reorder in sequence

3017------------------------------------------------------------------------------

3045------------------------------------------------------------------------------

src/Streamly/Streams/StreamD.hs+55

@@ -163,6 +163,8 @@ module Streamly.Streams.StreamD

163 , splitOn

164 , splitSuffixOn

165

166+ , splitInnerBy

167+

166 -- ** Substreams

168 -- ** Substreams

167 , isPrefixOf

169 , isPrefixOf

168 , isSubsequenceOf

170 , isSubsequenceOf

@@ -1628,6 +1630,59 @@ splitSuffixOn withSep patArr@Array{..} (Fold fstep initial done)

1628

1630

1629 stepOuter _ GO_DONE = return Stop

1631 stepOuter _ GO_DONE = return Stop

1630

1632

1633+data SplitState s arr

1634+ = SplitInitial s

1635+ | SplitBuffering s arr

1636+ | SplitSplitting s arr

1637+ | SplitYielding arr (SplitState s arr)

1638+ | SplitFinishing

1639+

1640+-- | Performs infix separator style splitting.

1641+{-# INLINE_NORMAL splitInnerBy #-}

1642+splitInnerBy

1643+ :: Monad m

1644+ => (f a -> m (f a, Maybe (f a))) -- splitter

1645+ -> (f a -> f a -> m (f a)) -- joiner

1646+ -> Stream m (f a)

1647+ -> Stream m (f a)

1648+splitInnerBy splitter joiner (Stream step1 state1) =

1649+ (Stream step (SplitInitial state1))

1650+

1651+ where

1652+

1653+ {-# INLINE_LATE step #-}

1654+ step gst (SplitInitial st) = do

1655+ r <- step1 gst st

1656+ case r of

1657+ Yield x s -> do

1658+ (x1, mx2) <- splitter x

1659+ return $ case mx2 of

1660+ Nothing -> Skip (SplitBuffering s x1)

1661+ Just x2 -> Skip (SplitYielding x1 (SplitSplitting s x2))

1662+ Skip s -> return $ Skip (SplitInitial s)

1663+ Stop -> return $ Stop

1664+

1665+ step gst (SplitBuffering st buf) = do

1666+ r <- step1 gst st

1667+ case r of

1668+ Yield x s -> do

1669+ (x1, mx2) <- splitter x

1670+ buf' <- joiner buf x1

1671+ return $ case mx2 of

1672+ Nothing -> Skip (SplitBuffering s buf')

1673+ Just x2 -> Skip (SplitYielding buf' (SplitSplitting s x2))

1674+ Skip s -> return $ Skip (SplitBuffering s buf)

1675+ Stop -> return $ Skip (SplitYielding buf SplitFinishing)

1676+

1677+ step _ (SplitSplitting st buf) = do

1678+ (x1, mx2) <- splitter buf

1679+ return $ case mx2 of

1680+ Nothing -> Skip $ SplitBuffering st x1

1681+ Just x2 -> Skip $ SplitYielding x1 (SplitSplitting st x2)

1682+

1683+ step _ (SplitYielding x next) = return $ Yield x next

1684+ step _ SplitFinishing = return $ Stop

1685+

1631------------------------------------------------------------------------------

1686------------------------------------------------------------------------------

1632-- Substreams

1687-- Substreams

1633------------------------------------------------------------------------------

1688------------------------------------------------------------------------------