d3faa5d · text · git.iko.soy

git.iko.soy/text

on master help sign in

Flesh out the implementation of lazy indices, with tests.

d3faa5dBryan O'Sullivan17 years ago

Data/Text/Lazy/Search.hs+30−16

compare with master

unifiedsplit

@@ -36,6 +36,7 @@ import qualified Data.Text.Array as A

36import Data.Int (Int64)

37import Data.Word (Word16, Word64)

38import qualified Data.Text.Internal as T

39+import qualified Data.Text as T

39import Data.Text.Fusion.Internal (PairS(..))

40import Data.Text.Fusion.Internal (PairS(..))

40import Data.Text.Lazy.Internal (Text(..))

41import Data.Text.Lazy.Internal (Text(..))

41import Data.Bits ((.|.), (.&.))

42import Data.Bits ((.|.), (.&.))

@@ -49,12 +50,12 @@ import Data.Text.UnsafeShift (shiftL)

49indices :: Text -- ^ Substring to search for (@needle@)

50indices :: Text -- ^ Substring to search for (@needle@)

50 -> Text -- ^ Text to search in (@haystack@)

51 -> Text -- ^ Text to search in (@haystack@)

51 -> [Int64]

52 -> [Int64]

52−indices needle Empty = []

53−indices needle@(Chunk n ns) haystack@(Chunk c cs)

53+indices needle@(Chunk n ns) haystack@(Chunk k ks)

54 | nlen <= 0 || ldiff < 0 = []

55+ | nlen == 1 = scanOne (nindex 0) 0 k ks

55− | otherwise = scan 0 0 c cs

56+ | otherwise = scan 0 0 k ks

56 where

57 where

57− scan !g !i x@(T.Text harr hoff l) xs

58+ scan !g !i x@(T.Text _ _ l) xs

58 | g > ldiff = []

59 | g > ldiff = []

59 | i >= m = case xs of

60 | i >= m = case xs of

60 Empty -> []

61 Empty -> []

@@ -79,17 +80,28 @@ indices needle@(Chunk n ns) haystack@(Chunk c cs)

79 nlast = nlen - 1

80 nlast = nlen - 1

80 nindex = index n ns

81 nindex = index n ns

81 z = foldChunks fin 0 needle

82 z = foldChunks fin 0 needle

82− where fin _ (T.Text narr noff nlen) = A.unsafeIndex narr (noff+nlen-1)

83+ where fin _ (T.Text farr foff flen) = A.unsafeIndex farr (foff+flen-1)

83 mask :*: skip = buildTable needle

84 mask :*: skip = buildTable needle

85+ scanOne c i (T.Text oarr ooff olen) os = go 0

86+ where

87+ go h | h >= olen = case os of

88+ Empty -> []

89+ Chunk y ys -> scanOne c (i+fromIntegral olen) y ys

90+ | on == c = i + fromIntegral h : go (h+1)

91+ | otherwise = go (h+1)

92+ where on = A.unsafeIndex oarr (ooff+h)

93+indices _ _ = []

85index :: T.Text -> Text -> Int64 -> Word16

95index :: T.Text -> Text -> Int64 -> Word16

86index (T.Text arr off len) xs i

96index (T.Text arr off len) xs i

87− | j <= len = A.unsafeIndex arr (off+j)

97+ | j < len = A.unsafeIndex arr (off+j)

88 | otherwise = case xs of

98 | otherwise = case xs of

99+ Empty | j == len -> 0

89− Empty -> error "empty"

100+ | otherwise -> error "empty"

90 Chunk c cs -> index c cs (i-fromIntegral len)

101 Chunk c cs -> index c cs (i-fromIntegral len)

91 where j = fromIntegral i

102 where j = fromIntegral i

103

104+swizzle :: Word16 -> Word64

93swizzle k = 1 `shiftL` (fromIntegral k .&. 0x3f)

105swizzle k = 1 `shiftL` (fromIntegral k .&. 0x3f)

106

95foldChunks :: (a -> T.Text -> a) -> a -> Text -> a

107foldChunks :: (a -> T.Text -> a) -> a -> Text -> a

@@ -97,23 +109,25 @@ foldChunks _ z Empty = z

97foldChunks f z (Chunk c cs) = let z' = f z c

109foldChunks f z (Chunk c cs) = let z' = f z c

98 in z' `seq` foldChunks f z' cs

110 in z' `seq` foldChunks f z' cs

111

112+wordLength :: Text -> Int64

100wordLength = foldChunks sumLength 0

113wordLength = foldChunks sumLength 0

101 where sumLength i (T.Text _ _ l) = i + fromIntegral l

114 where sumLength i (T.Text _ _ l) = i + fromIntegral l

102

115

103buildTable :: Text -> PairS Word64 Int64

116buildTable :: Text -> PairS Word64 Int64

104buildTable Empty = 0 :*: 0

117buildTable Empty = 0 :*: 0

105−buildTable needle@(Chunk c cs) = outer c cs 0 0 (nlen-2)

118+buildTable needle@(Chunk k ks) = outer k ks 0 0 0 (nlen-2)

106 where

119 where

107− outer x@(T.Text xarr xoff xlen) xs = go

120+ outer (T.Text xarr xoff xlen) xs = go

108 where

121 where

109− go !i !mask !skip

122+ go !(g::Int64) !i !mask !skip

110− | i >= xlen = case xs of

123+ | i >= xlast = case xs of

111− Empty -> (mask .|. swizzle z) :*: skip

124+ Empty -> (mask .|. swizzle z) :*: skip

112− Chunk y ys -> outer y ys 0 mask skip

125+ Chunk y ys -> outer y ys g 0 mask skip

113− | otherwise = go (i+1) (mask .|. swizzle c) skip'

126+ | otherwise = go (g+1) (i+1) (mask .|. swizzle c) skip'

114 where c = A.unsafeIndex xarr (xoff+i)

127 where c = A.unsafeIndex xarr (xoff+i)

115− skip' | c == z = nlen - fromIntegral i - 2

128+ skip' | c == z = nlen - fromIntegral g - 2

116 | otherwise = skip

129 | otherwise = skip

130+ xlast = xlen - 1

117 nlen = wordLength needle

131 nlen = wordLength needle

118 z = foldChunks fin 0 needle

132 z = foldChunks fin 0 needle

119− where fin _ (T.Text narr noff nlen) = A.unsafeIndex narr (noff+nlen-1)

133+ where fin _ (T.Text farr foff flen) = A.unsafeIndex farr (foff+flen-1)

tests/Properties.hs+5

@@ -28,6 +28,7 @@ import System.IO.Unsafe (unsafePerformIO)

28import Test.Framework (defaultMain, testGroup)

29import Test.Framework.Providers.QuickCheck (testProperty)

30import Data.Text.Search (indices)

31+import qualified Data.Text.Lazy.Search as S (indices)

31import qualified SlowFunctions as Slow

32import qualified SlowFunctions as Slow

33import QuickCheckUtils (NotEmpty(..), small)

34import QuickCheckUtils (NotEmpty(..), small)

@@ -486,7 +487,9 @@ sf_zipWith p c s = (L.zipWith c (L.filter p s) . L.filter p) `eqP` (unpackS . S

486t_zipWith c s = L.zipWith c s `eqP` (unpackS . T.zipWith c (packS s))

487t_zipWith c s = L.zipWith c s `eqP` (unpackS . T.zipWith c (packS s))

487tl_zipWith c s = L.zipWith c s `eqP` (unpackS . TL.zipWith c (packS s))

488tl_zipWith c s = L.zipWith c s `eqP` (unpackS . TL.zipWith c (packS s))

488

489

490+slow_indices = unsquare (\s -> (map fromIntegral . Slow.indices (T.pack s)) `eq` (Slow.lazyIndices (TL.pack s) . TL.fromChunks . (:[])))

489t_indices = unsquare (\s -> Slow.indices s `eq` indices s)

491t_indices = unsquare (\s -> Slow.indices s `eq` indices s)

492+tl_indices = unsquare (\s -> Slow.lazyIndices s `eq` S.indices s)

490t_indices_occurs t = unsquare (\ts -> let s = T.intercalate t ts

493t_indices_occurs t = unsquare (\ts -> let s = T.intercalate t ts

491 in Slow.indices t s == indices t s)

494 in Slow.indices t s == indices t s)

492

495

@@ -837,7 +840,9 @@ tests = [

837 testProperty "t_findIndex" t_findIndex,

840 testProperty "t_findIndex" t_findIndex,

838 testProperty "t_count" t_count,

841 testProperty "t_count" t_count,

839 testProperty "tl_count" tl_count,

842 testProperty "tl_count" tl_count,

843+ testProperty "slow_indices" slow_indices,

840 testProperty "t_indices" t_indices,

844 testProperty "t_indices" t_indices,

845+ testProperty "tl_indices" tl_indices,

841 testProperty "t_indices_occurs" t_indices_occurs

846 testProperty "t_indices_occurs" t_indices_occurs

842 ],

847 ],

843

848

tests/SlowFunctions.hs+14

@@ -3,10 +3,13 @@

3module SlowFunctions

4 (

5 indices

6+ , lazyIndices

6 , split

7 , split

7 ) where

8 ) where

10+import Data.Int (Int64)

9import qualified Data.Text as T

11import qualified Data.Text as T

12+import qualified Data.Text.Lazy as L

10import Data.Text.Internal (Text(..))

13import Data.Text.Internal (Text(..))

11import Data.Text.Unsafe (iter_, unsafeHead, unsafeTail)

14import Data.Text.Unsafe (iter_, unsafeHead, unsafeTail)

@@ -23,6 +26,17 @@ indices needle@(Text _narr _noff nlen) haystack@(Text harr hoff hlen)

23 where t = Text harr (hoff+i) (hlen-i)

26 where t = Text harr (hoff+i) (hlen-i)

24 d = iter_ haystack i

27 d = iter_ haystack i

29+lazyIndices :: L.Text -> L.Text -> [Int64]

30+lazyIndices needle haystack

31+ | L.null needle = []

32+ | otherwise = scan 0 haystack

33+ where

34+ scan !i hay

35+ | L.null hay = []

36+ | needle `L.isPrefixOf` hay = i : scan (i+n) (L.drop n hay)

37+ | otherwise = scan (i+1) (L.tail hay)

38+ n = L.length needle

39+

26split :: Text -- ^ Text to split on

40split :: Text -- ^ Text to split on

27 -> Text -- ^ Input text

41 -> Text -- ^ Input text

28 -> [Text]

42 -> [Text]