git.iko.soy/text
on master help sign in
Implement split using the new indices function.

--HG-- extra : convert_revision : 2ca7122cd3c46e4232ac0d3e57c1ff4db01bb893

commit949120b3e10aa42e33b0e6f7a61b44fff7783abb authorBryan O'Sullivan <bos@serpentine.com> dateTue 6 Oct 2009, 14:33 (17 years ago) parentd9395f2
5 files changed+61−40
compare with master
unifiedsplit
Data/Text.hs+12−16 Data/Text/Search.hs+1−19 tests/Makefile+2−2 tests/Properties.hs+6−3 tests/SlowFunctions.hs+40
Data/Text.hs+12−16
@@ -180,16 +180,15 @@ import qualified Data.List as L
180180import Data.Monoid (Monoid(..))
181181import Data.Word (Word16)
182182import Data.String (IsString(..))
183−
184183import qualified Data.Text.Fusion as S
185184import qualified Data.Text.Fusion.Common as S
186185import Data.Text.Fusion (stream, reverseStream, unstream)
187−
188186import Data.Text.Internal (Text(..), empty, text, textP)
189187import qualified Prelude as P
190188import Data.Text.Unsafe (iter, iter_, reverseIter, unsafeHead, unsafeTail)
191189import Data.Text.UnsafeChar (unsafeChr)
192190import qualified Data.Text.Encoding.Utf16 as U16
191+import Data.Text.Search (indices)
193192
194193-- $fusion
195194--
@@ -899,7 +898,7 @@ tails t | null t = [empty]
899898-- copies to create substrings; they just construct new 'Text's that
900899-- are slices of the original.
901900
902−-- | /O(m*n)/ Break a 'Text' into pieces separated by the first
901+-- | /O(m+n)/ Break a 'Text' into pieces separated by the first
903902-- 'Text' argument, consuming the delimiter. An empty delimiter is
904903-- invalid, and will cause an error to be raised.
905904--
@@ -913,21 +912,18 @@ tails t | null t = [empty]
913912--
914913-- > intercalate s . split s == id
915914-- > split (singleton c) == splitWith (==c)
916−split :: Text -- ^ Text to split on
917− -> Text -- ^ Input text
918− -> [Text]
919−split pat src0
915+--
916+-- In (unlikely) bad cases, this function's time complexity
917+-- degenerates towards /O(n*m)/.
918+split :: Text -> Text -> [Text]
919+split pat src@(Text arr _off len)
920920 | null pat = emptyError "split"
921− | l == 1 = splitWith (== (unsafeHead pat)) src0
922− | otherwise = go src0
921+ | l == 1 = splitWith (== unsafeHead pat) src
922+ | otherwise = chomp 0 (indices pat src)
923923 where
924− l = length pat
925− go src = search 0 src
926− where
927− search !n !s
928− | null s = [src] -- not found
929− | pat `isPrefixOf` s = take n src : go (drop l s)
930− | otherwise = search (n+1) (unsafeTail s)
924+ l = length pat
925+ chomp s (x:xs) = textP arr s (x-s) : chomp (x+l) xs
926+ chomp s [] = [textP arr s (len-s)]
931927{-# INLINE [1] split #-}
932928
933929{-# RULES
Data/Text/Search.hs+1−19
@@ -26,19 +26,17 @@
2626--
2727-- * F. Lundh: The Fast Search Algorithm.
2828-- <http://effbot.org/zone/stringlib.htm> (2006)
29+
2930module Data.Text.Search
3031 (
3132 indices
32− , slowIndices
3333 ) where
3434
35−import qualified Data.Text as T
3635import qualified Data.Text.Array as A
3736import Data.Word (Word64)
3837import Data.Text.Internal (Text(..))
3938import Data.Bits ((.|.), (.&.))
4039import Data.Text.UnsafeShift (shiftL)
41−import Data.Text.Unsafe (iter_)
4240
4341-- | /O(n+m)/ Find the offsets of all non-overlapping indices of
4442-- @needle@ within @haystack@. In (unlikely) bad cases, this
@@ -82,19 +80,3 @@ indices _needle@(Text narr noff nlen) _haystack@(Text harr hoff hlen)
8280 | hindex i == c = i : loop (i+1)
8381 | otherwise = loop (i+1)
8482{-# INLINE indices #-}
85−
86−-- | /O(n*m)/ Find the offsets of all non-overlapping indices of
87−-- @needle@ within @haystack@. Provided purely as a slower, reliable
88−-- counterpart to 'indices'.
89−slowIndices :: Text -- ^ Substring to search for (@needle@)
90− -> Text -- ^ Text to search in (@haystack@)
91− -> [Int]
92−slowIndices needle@(Text _narr _noff nlen) haystack@(Text harr hoff hlen)
93− | T.null needle = []
94− | otherwise = scan 0
95− where
96− scan i | i >= hlen = []
97− | needle `T.isPrefixOf` t = i : scan (i+nlen)
98− | otherwise = scan (i+d)
99− where t = Text harr (hoff+i) (hlen-i)
100− d = iter_ haystack i
tests/Makefile+2−2
@@ -28,11 +28,11 @@ else
2828 cd .. && cabal build
2929endif
3030
31−Properties.o: QuickCheckUtils.o
31+Properties.o: QuickCheckUtils.o SlowFunctions.o
3232
3333QuickCheckUtils.o: $(lib)
3434
35−qc: Properties.o QuickCheckUtils.o
35+qc: Properties.o QuickCheckUtils.o SlowFunctions.o
3636 $(ghc) $(ghc-flags) -threaded -o $@ $^ $(lib)
3737
3838sb: SearchBench.o
tests/Properties.hs+6−3
@@ -27,7 +27,8 @@ import Prelude hiding (replicate)
2727import System.IO.Unsafe (unsafePerformIO)
2828import Test.Framework (defaultMain, testGroup)
2929import Test.Framework.Providers.QuickCheck (testProperty)
30−import Data.Text.Search
30+import Data.Text.Search (indices)
31+import qualified SlowFunctions as Slow
3132
3233import QuickCheckUtils (NotEmpty(..), small)
3334
@@ -397,6 +398,7 @@ tl_inits = L.inits `eqP` (map unpackS . TL.inits)
397398t_tails = L.tails `eqP` (map unpackS . T.tails)
398399tl_tails = L.tails `eqP` (map unpackS . TL.tails)
399400
401+t_split_split p = T.split p `eq` Slow.split p
400402t_split_i (NotEmpty t) = id `eq` (T.intercalate t . T.split t)
401403tl_split_i (NotEmpty t) = id `eq` (TL.intercalate t . TL.split t)
402404t_splitTimes_i k (NotEmpty t) = id `eq` (T.intercalate t . T.splitTimes k t)
@@ -506,9 +508,9 @@ sf_zipWith p c s = (L.zipWith c (L.filter p s) . L.filter p) `eqP` (unpackS . S
506508t_zipWith c s = L.zipWith c s `eqP` (unpackS . T.zipWith c (packS s))
507509tl_zipWith c s = L.zipWith c s `eqP` (unpackS . TL.zipWith c (packS s))
508510
509−t_indices = unsquare (\s -> slowIndices s `eq` indices s)
511+t_indices = unsquare (\s -> Slow.indices s `eq` indices s)
510512t_indices_occurs t = unsquare (\ts -> let s = T.intercalate t ts
511− in slowIndices t s == indices t s)
513+ in Slow.indices t s == indices t s)
512514
513515-- Bit shifts.
514516shiftL w = forAll (choose (0,width-1)) $ \k -> Bits.shiftL w k == U.shiftL w k
@@ -800,6 +802,7 @@ tests = [
800802 ],
801803
802804 testGroup "breaking many" [
805+ testProperty "t_split_split" t_split_split,
803806 testProperty "t_split_i" t_split_i,
804807 testProperty "t_splitTimes_i" t_splitTimes_i,
805808 testProperty "tl_splitTimes_i" tl_splitTimes_i,
tests/SlowFunctions.hsnew file+40
@@ -0,0 +1,40 @@
1+{-# LANGUAGE BangPatterns #-}
2+
3+module SlowFunctions
4+ (
5+ indices
6+ , split
7+ ) where
8+
9+import qualified Data.Text as T
10+import Data.Text.Internal (Text(..))
11+import Data.Text.Unsafe (iter_, unsafeHead, unsafeTail)
12+
13+indices :: Text -- ^ Substring to search for (@needle@)
14+ -> Text -- ^ Text to search in (@haystack@)
15+ -> [Int]
16+indices needle@(Text _narr _noff nlen) haystack@(Text harr hoff hlen)
17+ | T.null needle = []
18+ | otherwise = scan 0
19+ where
20+ scan i | i >= hlen = []
21+ | needle `T.isPrefixOf` t = i : scan (i+nlen)
22+ | otherwise = scan (i+d)
23+ where t = Text harr (hoff+i) (hlen-i)
24+ d = iter_ haystack i
25+
26+split :: Text -- ^ Text to split on
27+ -> Text -- ^ Input text
28+ -> [Text]
29+split pat src0
30+ | T.null pat = error "split: empty"
31+ | l == 1 = T.splitWith (== (unsafeHead pat)) src0
32+ | otherwise = go src0
33+ where
34+ l = T.length pat
35+ go src = search 0 src
36+ where
37+ search !n !s
38+ | T.null s = [src] -- not found
39+ | pat `T.isPrefixOf` s = T.take n src : go (T.drop l s)
40+ | otherwise = search (n+1) (unsafeTail s)