on master help sign in
Add portable line splitter.
1eac48fBryan O'Sullivan
Data/Text.hs+27−8
@@ -110,6 +110,7 @@ module Data.Text
110
110
111 -- ** Breaking into lines and words
111 -- ** Breaking into lines and words
112 , lines
112 , lines
113+ , lines'
113 , words
114 , words
114 , unlines
115 , unlines
115 , unwords
116 , unwords
@@ -143,7 +144,7 @@ module Data.Text
143 ) where
144 ) where
144
145
145import Prelude (Char, Bool(..), Functor(..), Int, Maybe(..), String,
146import Prelude (Char, Bool(..), Functor(..), Int, Maybe(..), String,
146− Eq, (==), (++),
147+ Eq(..), (++),
147 Read(..), Show(..),
148 Read(..), Show(..),
148 (&&), (||), (+), (-), (<), (>), (<=), (>=), (.), ($),
149 (&&), (||), (+), (-), (<), (>), (<=), (>=), (.), ($),
149 not, return, otherwise)
150 not, return, otherwise)
@@ -904,16 +905,34 @@ words t@(Text arr off len) = loop 0 0
904
905
905-- | /O(n)/ Breaks a 'Text' up into a list of 'Text's at
906-- | /O(n)/ Breaks a 'Text' up into a list of 'Text's at
906-- newline 'Char's. The resulting strings do not contain newlines.
907-- newline 'Char's. The resulting strings do not contain newlines.
907−--
908lines :: Text -> [Text]
908lines :: Text -> [Text]
909−lines ps
909+lines ps | null ps = []
910− | null ps = []
911− | otherwise = case search ps of
910+ | otherwise = h : if null t
912− Nothing -> [ps]
911+ then []
913− Just n -> take n ps : lines (drop (n+1) ps)
912+ else lines (unsafeTail t)
914− where search = elemIndex '\n'
913+ where (h,t) = span (/= '\n') ps
915{-# INLINE lines #-}
914{-# INLINE lines #-}
916
915
916+-- | /O(n)/ Portably breaks a 'Text' up into a list of 'Text's at line
917+-- boundaries.
918+--
919+-- A line boundary is considered to be either a line feed, a carriage
920+-- return immediately followed by a line feed, or a carriage return.
921+-- This accounts for both Unix and Windows line ending conventions,
922+-- and for the old convention used on Mac OS 9 and earlier.
923+lines' :: Text -> [Text]
924+lines' ps | null ps = []
925+ | otherwise = h : case uncons t of
926+ Nothing -> []
927+ Just (c,t')
928+ | c == '\n' -> lines t'
929+ | c == '\r' -> case uncons t' of
930+ Just ('\n',t'') -> lines t''
931+ _ -> lines t'
932+ where (h,t) = span notEOL ps
933+ notEOL c = c /= '\n' && c /= '\r'
934+{-# INLINE lines' #-}
935+
917-- | /O(n)/ Joins lines, after appending a terminating newline to
936-- | /O(n)/ Joins lines, after appending a terminating newline to
918-- each.
937-- each.
919unlines :: [Text] -> Text
938unlines :: [Text] -> Text
tests/Properties.hs+9
@@ -153,6 +153,14 @@ prop_breakSubstringC c
153 (unpack2 . T.breakSubstring (T.singleton c))
153 (unpack2 . T.breakSubstring (T.singleton c))
154
154
155prop_lines = L.lines `eqP` (map unpack . T.lines)
155prop_lines = L.lines `eqP` (map unpack . T.lines)
156+prop_lines' = lines' `eqP` (map unpack . T.lines')
157+ where lines' "" = []
158+ lines' s = let (l, s') = break eol s
159+ in l : case s' of
160+ [] -> []
161+ ('\r':'\n':s'') -> lines' s''
162+ (_:s'') -> lines' s''
163+ eol c = c == '\r' || c == '\n'
156prop_words = L.words `eqP` (map unpack . T.words)
164prop_words = L.words `eqP` (map unpack . T.words)
157prop_unlines = L.unlines `eq` (unpack . T.unlines . map pack)
165prop_unlines = L.unlines `eq` (unpack . T.unlines . map pack)
158prop_unwords = L.unwords `eq` (unpack . T.unwords . map pack)
166prop_unwords = L.unwords `eq` (unpack . T.unwords . map pack)
@@ -272,6 +280,7 @@ tests = [
272 ("prop_breakSubstring_isInfixOf", mytest prop_breakSubstring_isInfixOf),
280 ("prop_breakSubstring_isInfixOf", mytest prop_breakSubstring_isInfixOf),
273
281
274 ("prop_lines", mytest prop_lines),
282 ("prop_lines", mytest prop_lines),
283+ ("prop_lines'", mytest prop_lines'),
275 ("prop_words", mytest prop_words),
284 ("prop_words", mytest prop_words),
276 ("prop_unlines", mytest prop_unlines),
285 ("prop_unlines", mytest prop_unlines),
277 ("prop_unwords", mytest prop_unwords),
286 ("prop_unwords", mytest prop_unwords),