949120b · text · git.iko.soy

git.iko.soy/text

on master help sign in

Implement split using the new indices function.

949120bBryan O'Sullivan17 years ago

Data/Text.hs+12−16

compare with master

unifiedsplit

@@ -180,16 +180,15 @@ import qualified Data.List as L

180import Data.Monoid (Monoid(..))

181import Data.Word (Word16)

182import Data.String (IsString(..))

183−

184import qualified Data.Text.Fusion as S

183import qualified Data.Text.Fusion as S

185import qualified Data.Text.Fusion.Common as S

184import qualified Data.Text.Fusion.Common as S

186import Data.Text.Fusion (stream, reverseStream, unstream)

185import Data.Text.Fusion (stream, reverseStream, unstream)

187−

188import Data.Text.Internal (Text(..), empty, text, textP)

186import Data.Text.Internal (Text(..), empty, text, textP)

189import qualified Prelude as P

187import qualified Prelude as P

190import Data.Text.Unsafe (iter, iter_, reverseIter, unsafeHead, unsafeTail)

188import Data.Text.Unsafe (iter, iter_, reverseIter, unsafeHead, unsafeTail)

191import Data.Text.UnsafeChar (unsafeChr)

189import Data.Text.UnsafeChar (unsafeChr)

192import qualified Data.Text.Encoding.Utf16 as U16

190import qualified Data.Text.Encoding.Utf16 as U16

191+import Data.Text.Search (indices)

193

192

194-- $fusion

193-- $fusion

195--

194--

@@ -899,7 +898,7 @@ tails t | null t = [empty]

899-- copies to create substrings; they just construct new 'Text's that

898-- copies to create substrings; they just construct new 'Text's that

900-- are slices of the original.

899-- are slices of the original.

901

900

902−-- | /O(m*n)/ Break a 'Text' into pieces separated by the first

901+-- | /O(m+n)/ Break a 'Text' into pieces separated by the first

903-- 'Text' argument, consuming the delimiter. An empty delimiter is

902-- 'Text' argument, consuming the delimiter. An empty delimiter is

904-- invalid, and will cause an error to be raised.

903-- invalid, and will cause an error to be raised.

905--

904--

@@ -913,21 +912,18 @@ tails t | null t = [empty]

913--

912--

914-- > intercalate s . split s == id

913-- > intercalate s . split s == id

915-- > split (singleton c) == splitWith (==c)

914-- > split (singleton c) == splitWith (==c)

915+--

916+-- In (unlikely) bad cases, this function's time complexity

917+-- degenerates towards /O(n*m)/.

916−split :: Text -- ^ Text to split on

918+split :: Text -> Text -> [Text]

917− -> Text -- ^ Input text

918− -> [Text]

919−split pat src0

919+split pat src@(Text arr _off len)

920 | null pat = emptyError "split"

921− | l == 1 = splitWith (== (unsafeHead pat)) src0

921+ | l == 1 = splitWith (== unsafeHead pat) src

922− | otherwise = go src0

922+ | otherwise = chomp 0 (indices pat src)

923 where

924− l = length pat

924+ l = length pat

925− go src = search 0 src

925+ chomp s (x:xs) = textP arr s (x-s) : chomp (x+l) xs

926− where

926+ chomp s [] = [textP arr s (len-s)]

927− search !n !s

928− | null s = [src] -- not found

929− | pat `isPrefixOf` s = take n src : go (drop l s)

930− | otherwise = search (n+1) (unsafeTail s)

931{-# INLINE [1] split #-}

927{-# INLINE [1] split #-}

932

928

933{-# RULES

929{-# RULES

Data/Text/Search.hs+1−19

@@ -26,19 +26,17 @@

26--

27-- * F. Lundh: The Fast Search Algorithm.

28-- <http://effbot.org/zone/stringlib.htm> (2006)

29+

29module Data.Text.Search

30module Data.Text.Search

30 (

31 (

31 indices

32 indices

32− , slowIndices

33 ) where

35−import qualified Data.Text as T

36import qualified Data.Text.Array as A

35import qualified Data.Text.Array as A

37import Data.Word (Word64)

36import Data.Word (Word64)

38import Data.Text.Internal (Text(..))

37import Data.Text.Internal (Text(..))

39import Data.Bits ((.|.), (.&.))

38import Data.Bits ((.|.), (.&.))

40import Data.Text.UnsafeShift (shiftL)

39import Data.Text.UnsafeShift (shiftL)

41−import Data.Text.Unsafe (iter_)

43-- | /O(n+m)/ Find the offsets of all non-overlapping indices of

41-- | /O(n+m)/ Find the offsets of all non-overlapping indices of

44-- @needle@ within @haystack@. In (unlikely) bad cases, this

42-- @needle@ within @haystack@. In (unlikely) bad cases, this

@@ -82,19 +80,3 @@ indices _needle@(Text narr noff nlen) _haystack@(Text harr hoff hlen)

82 | hindex i == c = i : loop (i+1)

80 | hindex i == c = i : loop (i+1)

83 | otherwise = loop (i+1)

81 | otherwise = loop (i+1)

84{-# INLINE indices #-}

82{-# INLINE indices #-}

85−

86−-- | /O(n*m)/ Find the offsets of all non-overlapping indices of

87−-- @needle@ within @haystack@. Provided purely as a slower, reliable

88−-- counterpart to 'indices'.

89−slowIndices :: Text -- ^ Substring to search for (@needle@)

90− -> Text -- ^ Text to search in (@haystack@)

91− -> [Int]

92−slowIndices needle@(Text _narr _noff nlen) haystack@(Text harr hoff hlen)

93− | T.null needle = []

94− | otherwise = scan 0

95− where

96− scan i | i >= hlen = []

97− | needle `T.isPrefixOf` t = i : scan (i+nlen)

98− | otherwise = scan (i+d)

99− where t = Text harr (hoff+i) (hlen-i)

100− d = iter_ haystack i

tests/Makefile+2−2

@@ -28,11 +28,11 @@ else

28 cd .. && cabal build

29endif

31−Properties.o: QuickCheckUtils.o

31+Properties.o: QuickCheckUtils.o SlowFunctions.o

33QuickCheckUtils.o: $(lib)

35−qc: Properties.o QuickCheckUtils.o

35+qc: Properties.o QuickCheckUtils.o SlowFunctions.o

36 $(ghc) $(ghc-flags) -threaded -o $@ $^ $(lib)

38sb: SearchBench.o

tests/Properties.hs+6−3

@@ -27,7 +27,8 @@ import Prelude hiding (replicate)

27import System.IO.Unsafe (unsafePerformIO)

28import Test.Framework (defaultMain, testGroup)

29import Test.Framework.Providers.QuickCheck (testProperty)

30−import Data.Text.Search

30+import Data.Text.Search (indices)

31+import qualified SlowFunctions as Slow

32import QuickCheckUtils (NotEmpty(..), small)

33import QuickCheckUtils (NotEmpty(..), small)

@@ -397,6 +398,7 @@ tl_inits = L.inits `eqP` (map unpackS . TL.inits)

397t_tails = L.tails `eqP` (map unpackS . T.tails)

398t_tails = L.tails `eqP` (map unpackS . T.tails)

398tl_tails = L.tails `eqP` (map unpackS . TL.tails)

399tl_tails = L.tails `eqP` (map unpackS . TL.tails)

399

400

401+t_split_split p = T.split p `eq` Slow.split p

400t_split_i (NotEmpty t) = id `eq` (T.intercalate t . T.split t)

402t_split_i (NotEmpty t) = id `eq` (T.intercalate t . T.split t)

401tl_split_i (NotEmpty t) = id `eq` (TL.intercalate t . TL.split t)

403tl_split_i (NotEmpty t) = id `eq` (TL.intercalate t . TL.split t)

402t_splitTimes_i k (NotEmpty t) = id `eq` (T.intercalate t . T.splitTimes k t)

404t_splitTimes_i k (NotEmpty t) = id `eq` (T.intercalate t . T.splitTimes k t)

@@ -506,9 +508,9 @@ sf_zipWith p c s = (L.zipWith c (L.filter p s) . L.filter p) `eqP` (unpackS . S

506t_zipWith c s = L.zipWith c s `eqP` (unpackS . T.zipWith c (packS s))

508t_zipWith c s = L.zipWith c s `eqP` (unpackS . T.zipWith c (packS s))

507tl_zipWith c s = L.zipWith c s `eqP` (unpackS . TL.zipWith c (packS s))

509tl_zipWith c s = L.zipWith c s `eqP` (unpackS . TL.zipWith c (packS s))

508

510

509−t_indices = unsquare (\s -> slowIndices s `eq` indices s)

511+t_indices = unsquare (\s -> Slow.indices s `eq` indices s)

510t_indices_occurs t = unsquare (\ts -> let s = T.intercalate t ts

512t_indices_occurs t = unsquare (\ts -> let s = T.intercalate t ts

511− in slowIndices t s == indices t s)

513+ in Slow.indices t s == indices t s)

512

514

513-- Bit shifts.

515-- Bit shifts.

514shiftL w = forAll (choose (0,width-1)) $ \k -> Bits.shiftL w k == U.shiftL w k

516shiftL w = forAll (choose (0,width-1)) $ \k -> Bits.shiftL w k == U.shiftL w k

@@ -800,6 +802,7 @@ tests = [

800 ],

802 ],

801

803

802 testGroup "breaking many" [

804 testGroup "breaking many" [

805+ testProperty "t_split_split" t_split_split,

803 testProperty "t_split_i" t_split_i,

806 testProperty "t_split_i" t_split_i,

804 testProperty "t_splitTimes_i" t_splitTimes_i,

807 testProperty "t_splitTimes_i" t_splitTimes_i,

805 testProperty "tl_splitTimes_i" tl_splitTimes_i,

808 testProperty "tl_splitTimes_i" tl_splitTimes_i,

tests/SlowFunctions.hsnew file+40

@@ -0,0 +1,40 @@

1+{-# LANGUAGE BangPatterns #-}

3+module SlowFunctions

4+ (

5+ indices

6+ , split

7+ ) where

9+import qualified Data.Text as T

10+import Data.Text.Internal (Text(..))

11+import Data.Text.Unsafe (iter_, unsafeHead, unsafeTail)

12+

13+indices :: Text -- ^ Substring to search for (@needle@)

14+ -> Text -- ^ Text to search in (@haystack@)

15+ -> [Int]

16+indices needle@(Text _narr _noff nlen) haystack@(Text harr hoff hlen)

17+ | T.null needle = []

18+ | otherwise = scan 0

19+ where

20+ scan i | i >= hlen = []

21+ | needle `T.isPrefixOf` t = i : scan (i+nlen)

22+ | otherwise = scan (i+d)

23+ where t = Text harr (hoff+i) (hlen-i)

24+ d = iter_ haystack i

25+

26+split :: Text -- ^ Text to split on

27+ -> Text -- ^ Input text

28+ -> [Text]

29+split pat src0

30+ | T.null pat = error "split: empty"

31+ | l == 1 = T.splitWith (== (unsafeHead pat)) src0

32+ | otherwise = go src0

33+ where

34+ l = T.length pat

35+ go src = search 0 src

36+ where

37+ search !n !s

38+ | T.null s = [src] -- not found

39+ | pat `T.isPrefixOf` s = T.take n src : go (T.drop l s)

40+ | otherwise = search (n+1) (unsafeTail s)