ab6587b · streamly · git.iko.soy

examples/FileSinkServer.hs+1−1

compare with master

unifiedsplit

@@ -10,7 +10,7 @@ import Network.Socket (close)

10import System.Environment (getArgs)

11

12import Streamly

13−import Streamly.Data.String

13+import Streamly.Data.Unicode.Stream

14import qualified Streamly.FileSystem.Handle as FH

15import qualified Streamly.Memory.Array as A

16import qualified Streamly.Network.Socket as NS

examples/WordClassifier.hs+1−1

@@ -16,7 +16,7 @@ import Data.IORef

16import qualified Data.List as List

17import qualified Data.Ord as Ord

18import Foreign.Storable (Storable(..))

19−import qualified Streamly.Data.String as Streamly

19+import qualified Streamly.Data.Unicode.Stream as Streamly

20import qualified Streamly.FileSystem.Handle as FH

21import qualified Streamly.Data.Fold as FL

22import qualified Streamly.Internal.Data.Fold as IFL

examples/WordCount.hs+1−1

@@ -38,7 +38,7 @@ import System.Environment (getArgs)

38import System.IO (Handle, openFile, IOMode(..))

39

40import qualified Streamly as S

41−import qualified Streamly.Data.String as S

41+import qualified Streamly.Data.Unicode.Stream as S

42import qualified Streamly.FileSystem.Handle as FH

43import qualified Streamly.Memory.Array as A

44import qualified Streamly.Prelude as S

src/Streamly/Benchmark/FileIO/Array.hs+1−1

@@ -45,7 +45,7 @@ import Prelude hiding (last)

45import qualified Streamly.FileSystem.Handle as FH

46import qualified Streamly.Memory.Array as A

47import qualified Streamly.Prelude as S

48−import qualified Streamly.Data.String as SS

48+import qualified Streamly.Data.Unicode.Stream as SS

49

50import qualified Streamly.Internal.FileSystem.Handle as IFH

51import qualified Streamly.Internal.Memory.Array as IA

src/Streamly/Benchmark/FileIO/Stream.hs+1−1

@@ -77,7 +77,7 @@ import qualified Streamly.Internal.Memory.Array.Types as AT

77import qualified Streamly.Prelude as S

78import qualified Streamly.Data.Fold as FL

79-- import qualified Streamly.Internal.Data.Fold as IFL

80−import qualified Streamly.Data.String as SS

80+import qualified Streamly.Data.Unicode.Stream as SS

81import qualified Streamly.Internal.Data.Unfold as IUF

82import qualified Streamly.Internal.Prelude as IP

83import qualified Streamly.Streams.StreamD as D

src/Streamly/Data/Unicode/Stream.hsnew file+88

@@ -0,0 +1,88 @@

1+{-# LANGUAGE FlexibleContexts #-}

2+-- |

3+-- Module : Streamly.Data.Unicode.Stream

5+--

6+-- License : BSD3

7+-- Maintainer : streamly@composewell.com

8+-- Stability : experimental

9+-- Portability : GHC

10+--

11+-- A unicode string can be represented either as a stream of 'Char' e.g.

12+-- 'SerialT' 'Identity' 'Char' or as 'Array' 'Char'. Unicode text processing

13+-- can be done efficiently by applying stream operations and folds on the

14+-- stream of 'Char'. When using 'Array' 'Char' direct array operations can be

15+-- applied where available or the array can be read into a stream and then

16+-- processed using stream operations. Use 'Array' 'Char' when you need to store

17+-- or buffer strings temporarily in memory. Streams in 'Identity' monad and

18+-- 'Array' 'Char' are instances of 'IsString' and 'IsList', therefore,

19+-- 'OverloadedStrings' and 'OverloadedLists' extensions can be used for

20+-- convenience.

21+--

22+-- 'Array' 'Char' is usually perfectly fine to buffer short to medium or even

23+-- large amounts of text in memory. Also, it is computationally efficient as

24+-- there is no encoding/decoding involved. We recommend using true streaming

25+-- operations to avoid buffering large amounts of data as long as possible e.g.

26+-- use `foldLines` instead of `lines`. However, if for some reason you are

27+-- buffering very large amounts of text in memory and are worried about space

28+-- efficiency you can use 'encodeUtf8' on the stream to convert it to a utf8

29+-- encoded 'Array' 'Word8'.

30+--

31+-- Please note the following:

32+--

33+-- * Case conversion: Some unicode characters translate to more than one code

34+-- point on case conversion. The 'toUpper' and 'toLower' functions in @base@

35+-- package do not handle such characters. Therefore, operations like @map

36+-- toUpper@ on a character stream or character array may not always perform

37+-- correct conversion.

38+-- * String comparison: In some cases, visually identical strings may have

39+-- different unicode representations, therefore, a character stream or

40+-- character array cannot be directly compared. A normalized comparison may be

41+-- needed to check string equivalence correctly.

42+

43+module Streamly.Data.Unicode.Stream

44+ (

45+ -- * Construction (Decoding)

46+ decodeChar8

47+ , decodeUtf8

48+ , decodeUtf8Lenient

49+ , D.DecodeError(..)

50+ , D.DecodeState

51+ , D.CodePoint

52+ , decodeUtf8Either

53+ , resumeDecodeUtf8Either

54+ , decodeUtf8Arrays

55+ , decodeUtf8ArraysLenient

56+

57+ -- * Elimination (Encoding)

58+ , encodeChar8

59+ , encodeChar8Unchecked

60+ , encodeUtf8

61+{-

62+ -- * Unicode aware operations

63+ , toCaseFold

64+ , toLower

65+ , toUpper

66+ , toTitle

67+

68+ -- * Operations on character strings

69+ , strip -- (dropAround isSpace)

70+ , stripEnd-}

71+ -- * Transformation

72+ , stripStart

73+ , foldLines

74+ , foldWords

75+ , unfoldLines

76+ , unfoldWords

77+

78+ -- * Streams of Strings

79+ , lines

80+ , words

81+ , unlines

82+ , unwords

83+ )

84+where

85+

86+import Streamly.Internal.Data.Unicode.Stream

87+import Prelude hiding (lines, words, unlines, unwords)

88+import qualified Streamly.Streams.StreamD as D

src/Streamly/Data/String.hs → src/Streamly/Internal/Data/Unicode/Stream.hsrenamed+3−41

@@ -1,52 +1,14 @@

1{-# LANGUAGE FlexibleContexts #-}

2-- |

3−-- Module : Streamly.Data.String

3+-- Module : Streamly.Data.Internal.Unicode.Stream

5--

6-- License : BSD3

7−-- Maintainer : harendra.kumar@gmail.com

7+-- Maintainer : streamly@composewell.com

8-- Stability : experimental

9-- Portability : GHC

10--

11−-- A unicode string can be represented either as a stream of 'Char' e.g.

11+module Streamly.Internal.Data.Unicode.Stream

12−-- 'SerialT' 'Identity' 'Char' or as 'Array' 'Char'. Unicode text processing

13−-- can be done efficiently by applying stream operations and folds on the

14−-- stream of 'Char'. When using 'Array' 'Char' direct array operations can be

15−-- applied where available or the array can be read into a stream and then

16−-- processed using stream operations. Use 'Array' 'Char' when you need to store

17−-- or buffer strings temporarily in memory. Streams in 'Identity' monad and

18−-- 'Array' 'Char' are instances of 'IsString' and 'IsList', therefore,

19−-- 'OverloadedStrings' and 'OverloadedLists' extensions can be used for

20−-- convenience.

21−--

22−-- 'Array' 'Char' is usually perfectly fine to buffer short to medium or even

23−-- large amounts of text in memory. Also, it is computationally efficient as

24−-- there is no encoding/decoding involved. We recommend using true streaming

25−-- operations to avoid buffering large amounts of data as long as possible e.g.

26−-- use `foldLines` instead of `lines`. However, if for some reason you are

27−-- buffering very large amounts of text in memory and are worried about space

28−-- efficiency you can use 'encodeUtf8' on the stream to convert it to a utf8

29−-- encoded 'Array' 'Word8'.

30−--

31−-- Please note the following:

32−--

33−-- * Case conversion: Some unicode characters translate to more than one code

34−-- point on case conversion. The 'toUpper' and 'toLower' functions in @base@

35−-- package do not handle such characters. Therefore, operations like @map

36−-- toUpper@ on a character stream or character array may not always perform

37−-- correct conversion.

38−-- * String comparison: In some cases, visually identical strings may have

39−-- different unicode representations, therefore, a character stream or

40−-- character array cannot be directly compared. A normalized comparison may be

41−-- needed to check string equivalence correctly.

42−

43−-- See "Streamly.Internal.Data.List", <src/docs/streamly-vs-lists.md> for more details and

44−-- <src/test/PureStreams.hs> for comprehensive usage examples.

45−

46−-- TBD write a note on performance comparison with text, bytestrings, lists and

47−-- vector.

48−--

49−module Streamly.Data.String

50 (

12 (

51 -- * Construction (Decoding)

13 -- * Construction (Decoding)

52 decodeChar8

14 decodeChar8

streamly.cabal+2−1

@@ -283,7 +283,7 @@ library

283 exposed-modules: Streamly.Prelude

284 , Streamly

285 , Streamly.Data.Fold

286− , Streamly.Data.String

286+ , Streamly.Data.Unicode.Stream

287

288 -- IO devices

289 , Streamly.Memory.Array

@@ -312,6 +312,7 @@ library

312 , Streamly.Internal.Data.List

313 , Streamly.Internal.FileSystem.Handle

314 , Streamly.Internal.FileSystem.File

315+ , Streamly.Internal.Data.Unicode.Stream

315 , Streamly.Internal.Prelude

316 , Streamly.Internal.Prelude

316 if !impl(ghcjs)

317 if !impl(ghcjs)

317 exposed-modules:

318 exposed-modules:

test/String.hs+1−1

@@ -15,7 +15,7 @@ import Test.Hspec as H

15import qualified Streamly.Memory.Array as A

16import qualified Streamly.Internal.Memory.ArrayStream as AS

17import qualified Streamly.Prelude as S

18−import qualified Streamly.Data.String as SS

18+import qualified Streamly.Data.Unicode.Stream as SS

19

20-- Coverage build takes too long with default number of tests

21{-