@@ -2635,46 +2635,55 @@ byteEq arr1 arr2 = fmap (EQ ==) $ byteCmp arr1 arr2
26352635-- Compact
26362636-------------------------------------------------------------------------------
26372637
2638−-- | Parser @pCompactLE n@ coalesces adjacent arrays in the input stream
2639−-- only if the combined size would be less than or equal to n.
2638+-- Note: LE versions avoid an extra copy compared to GE. LE parser trades
2639+-- backtracking one array in lieu of avoiding a copy. However, LE and GE both
2640+-- can leave some memory unused. They can split the last array to fit it
2641+-- exactly in the space.
2642+
2643+-- | Parser @pCompactLE maxElems@ coalesces adjacent arrays in the input stream
2644+-- only if the combined size would be less than or equal to @maxElems@
2645+-- elements. Note that it won't split an array if the original array is already
2646+-- larger than maxElems.
2647+--
2648+-- @maxElems@ must be greater than 0.
26402649--
26412650-- /Internal/
26422651{-# INLINE_NORMAL pCompactLE #-}
26432652pCompactLE ::
26442653 forall m a. (MonadIO m, Unbox a)
26452654 => Int -> Parser (MutArray a) m (MutArray a)
2646−pCompactLE n = Parser step initial extract
2655+pCompactLE maxElems = Parser step initial extract
26472656
26482657 where
26492658
2650− nBytes = n * SIZE_OF(a)
2659+ maxBytes = maxElems * SIZE_OF(a)
26512660
26522661 functionName = "Streamly.Internal.Data.MutArray.pCompactLE"
26532662
26542663 initial =
26552664 return
2656− $ if n <= 0
2665+ $ if maxElems <= 0
26572666 then error
26582667 $ functionName
26592668 ++ ": the size of arrays ["
2660− ++ show n ++ "] must be a natural number"
2669+ ++ show maxElems ++ "] must be a natural number"
26612670 else Parser.IPartial Nothing
26622671
26632672 step Nothing arr =
26642673 return
26652674 $ let len = byteLength arr
2666− in if len >= nBytes
2675+ in if len >= maxBytes
26672676 then Parser.Done 0 arr
26682677 else Parser.Partial 0 (Just arr)
2678+ -- XXX Split the last array to use the space more compactly.
26692679 step (Just buf) arr =
26702680 let len = byteLength buf + byteLength arr
2671− in if len > nBytes
2681+ in if len > maxBytes
26722682 then return $ Parser.Done 1 buf
26732683 else do
2674− -- XXX check this in the Nothing case
26752684 buf1 <-
2676− if byteCapacity buf < nBytes
2677− then liftIO $ realloc nBytes buf
2685+ if byteCapacity buf < maxBytes
2686+ then realloc maxBytes buf
26782687 else return buf
26792688 buf2 <- spliceUnsafe buf1 arr
26802689 return $ Parser.Partial 0 (Just buf2)
@@ -2693,48 +2702,54 @@ data SpliceState s arr
26932702-- immutable array never has additional space so a new array is allocated
26942703-- instead of mutating it.
26952704
2696−-- | Scan @rCompactLE n@ coalesces adjacent arrays in the input stream
2697−-- only if the combined size would be less than or equal to n.
2705+-- | Scan @rCompactLE maxElems@ coalesces adjacent arrays in the input stream
2706+-- only if the combined size would be less than or equal to @maxElems@
2707+-- elements. Note that it won't split an array if the original array is already
2708+-- larger than maxElems.
2709+--
2710+-- @maxElems@ must be greater than 0.
26982711--
26992712-- /Internal/
27002713{-# INLINE_NORMAL rCompactLE #-}
2701−rCompactLE :: (MonadIO m, Unbox a)
2714+rCompactLE :: forall m a. (MonadIO m, Unbox a)
27022715 => Int -> D.Stream m (MutArray a) -> D.Stream m (MutArray a)
2703−rCompactLE n (D.Stream step state) =
2716+rCompactLE maxElems (D.Stream step state) =
27042717 D.Stream step' (SpliceInitial state)
27052718
27062719 where
27072720
2721+ maxBytes = maxElems * SIZE_OF(a)
2722+
27082723 functionName = "Streamly.Internal.Data.MutArray.rCompactLE"
27092724
27102725 {-# INLINE_LATE step' #-}
27112726 step' gst (SpliceInitial st) = do
2712− when (n <= 0) $
2727+ when (maxElems <= 0) $
27132728 -- XXX we can pass the module string from the higher level API
2714− error $ functionName ++ ": the size of arrays [" ++ show n
2729+ error $ functionName ++ ": the size of arrays [" ++ show maxElems
27152730 ++ "] must be a natural number"
27162731 r <- step gst st
27172732 case r of
27182733 D.Yield arr s -> return $
27192734 let len = byteLength arr
2720− in if len >= n
2735+ in if len >= maxBytes
27212736 then D.Skip (SpliceYielding arr (SpliceInitial s))
27222737 else D.Skip (SpliceBuffering s arr)
27232738 D.Skip s -> return $ D.Skip (SpliceInitial s)
27242739 D.Stop -> return D.Stop
27252740
2741+ -- XXX Split the last array to use the space more compactly.
27262742 step' gst (SpliceBuffering st buf) = do
27272743 r <- step gst st
27282744 case r of
27292745 D.Yield arr s -> do
27302746 let len = byteLength buf + byteLength arr
2731− if len > n
2747+ if len > maxBytes
27322748 then return $
27332749 D.Skip (SpliceYielding buf (SpliceBuffering s arr))
27342750 else do
2735− -- XXX check this in the SpliceInitial case
2736− buf1 <- if byteCapacity buf < n
2737− then liftIO $ realloc n buf
2751+ buf1 <- if byteCapacity buf < maxBytes
2752+ then realloc maxBytes buf
27382753 else return buf
27392754 buf2 <- spliceUnsafe buf1 arr
27402755 return $ D.Skip (SpliceBuffering s buf2)
@@ -2745,44 +2760,45 @@ rCompactLE n (D.Stream step state) =
27452760
27462761 step' _ (SpliceYielding arr next) = return $ D.Yield arr next
27472762
2748−-- | Fold @fCompactGE n@ coalesces adjacent arrays in the input stream
2749−-- until the size becomes greater than or equal to n.
2763+-- | Fold @fCompactGE minElems@ coalesces adjacent arrays in the input stream
2764+-- until the size becomes greater than or equal to @minElems@.
27502765--
27512766{-# INLINE_NORMAL fCompactGE #-}
27522767fCompactGE ::
27532768 forall m a. (MonadIO m, Unbox a)
27542769 => Int -> FL.Fold m (MutArray a) (MutArray a)
2755−fCompactGE n = Fold step initial extract extract
2770+fCompactGE minElems = Fold step initial extract extract
27562771
27572772 where
27582773
2759− nBytes = n * SIZE_OF(a)
2774+ minBytes = minElems * SIZE_OF(a)
27602775
27612776 functionName = "Streamly.Internal.Data.MutArray.fCompactGE"
27622777
27632778 initial =
27642779 return
2765− $ if n < 0
2780+ $ if minElems < 0
27662781 then error
27672782 $ functionName
27682783 ++ ": the size of arrays ["
2769− ++ show n ++ "] must be a natural number"
2784+ ++ show minElems ++ "] must be a natural number"
27702785 else FL.Partial Nothing
27712786
27722787 step Nothing arr =
27732788 return
27742789 $ let len = byteLength arr
2775− in if len >= nBytes
2790+ in if len >= minBytes
27762791 then FL.Done arr
27772792 else FL.Partial (Just arr)
2793+ -- XXX Buffer arrays as a list to avoid copy and reallocations
27782794 step (Just buf) arr = do
27792795 let len = byteLength buf + byteLength arr
27802796 buf1 <-
27812797 if byteCapacity buf < len
2782− then liftIO $ realloc (max len nBytes) buf
2798+ then realloc (max len minBytes) buf
27832799 else return buf
27842800 buf2 <- spliceUnsafe buf1 arr
2785− if len >= n
2801+ if len >= minBytes
27862802 then return $ FL.Done buf2
27872803 else return $ FL.Partial (Just buf2)
27882804
@@ -2794,59 +2810,54 @@ fCompactGE n = Fold step initial extract extract
27942810-- >>> lCompactGE n = Fold.many (MutArray.fCompactGE n)
27952811--
27962812{-# INLINE_NORMAL lCompactGE #-}
2797−lCompactGE :: (MonadIO m, Unbox a)
2813+lCompactGE :: forall m a. (MonadIO m, Unbox a)
27982814 => Int -> Fold m (MutArray a) () -> Fold m (MutArray a) ()
27992815-- lCompactGE n = Fold.many (fCompactGE n)
2800−lCompactGE n (Fold step1 initial1 _ final1) =
2816+lCompactGE minElems (Fold step1 initial1 _ final1) =
28012817 Fold step initial extract final
28022818
28032819 where
28042820
2821+ minBytes = minElems * SIZE_OF(a)
2822+
28052823 functionName = "Streamly.Internal.Data.MutArray.lCompactGE"
28062824
28072825 initial = do
2808− when (n <= 0) $
2826+ when (minElems <= 0) $
28092827 -- XXX we can pass the module string from the higher level API
28102828 error $ functionName ++ ": the size of arrays ["
2811− ++ show n ++ "] must be a natural number"
2829+ ++ show minElems ++ "] must be a natural number"
28122830
28132831 r <- initial1
28142832 return $ first (Tuple' Nothing) r
28152833
2816− step (Tuple' Nothing r1) arr =
2817− let len = byteLength arr
2818− in if len >= n
2819− then do
2820− r <- step1 r1 arr
2821− case r of
2822− FL.Done _ -> return $ FL.Done ()
2823− FL.Partial s -> do
2824− _ <- final1 s
2825− res <- initial1
2826− return $ first (Tuple' Nothing) res
2827− else return $ FL.Partial $ Tuple' (Just arr) r1
2828−
2829− step (Tuple' (Just buf) r1) arr = do
2830− let len = byteLength buf + byteLength arr
2831− buf1 <- if byteCapacity buf < len
2832− then liftIO $ realloc (max n len) buf
2833− else return buf
2834− buf2 <- spliceUnsafe buf1 arr
2835−
2836− -- XXX this is common in both the equations of step
2837− if len >= n
2834+ {-# INLINE runInner #-}
2835+ runInner len acc buf =
2836+ if len >= minBytes
28382837 then do
2839− r <- step1 r1 buf2
2838+ r <- step1 acc buf
28402839 case r of
28412840 FL.Done _ -> return $ FL.Done ()
28422841 FL.Partial s -> do
28432842 _ <- final1 s
28442843 res <- initial1
28452844 return $ first (Tuple' Nothing) res
2846− else return $ FL.Partial $ Tuple' (Just buf2) r1
2845+ else return $ FL.Partial $ Tuple' (Just buf) acc
2846+
2847+ step (Tuple' Nothing r1) arr =
2848+ runInner (byteLength arr) r1 arr
2849+
2850+ -- XXX Buffer arrays as a list to avoid copy and reallocations
2851+ step (Tuple' (Just buf) r1) arr = do
2852+ let len = byteLength buf + byteLength arr
2853+ buf1 <- if byteCapacity buf < len
2854+ then realloc (max minBytes len) buf
2855+ else return buf
2856+ buf2 <- spliceUnsafe buf1 arr
2857+ runInner len r1 buf2
28472858
28482859 -- XXX Several folds do extract >=> final, therefore, we need to make final
2849− -- return "m b" rather than using extract post it if we want extract to be
2860+ -- return "m b" rather than using extract post it if we want extract to be
28502861 -- partial.
28512862 --
28522863 -- extract forces the pending buffer to be sent to the fold which is not