@@ -0,0 +1,88 @@
1+{-# LANGUAGE FlexibleContexts #-}
2+-- |
3+-- Module : Streamly.Data.Unicode.Stream
4+-- Copyright : (c) 2018 Composewell Technologies
5+--
6+-- License : BSD3
7+-- Maintainer : streamly@composewell.com
8+-- Stability : experimental
9+-- Portability : GHC
10+--
11+-- A unicode string can be represented either as a stream of 'Char' e.g.
12+-- 'SerialT' 'Identity' 'Char' or as 'Array' 'Char'. Unicode text processing
13+-- can be done efficiently by applying stream operations and folds on the
14+-- stream of 'Char'. When using 'Array' 'Char' direct array operations can be
15+-- applied where available or the array can be read into a stream and then
16+-- processed using stream operations. Use 'Array' 'Char' when you need to store
17+-- or buffer strings temporarily in memory. Streams in 'Identity' monad and
18+-- 'Array' 'Char' are instances of 'IsString' and 'IsList', therefore,
19+-- 'OverloadedStrings' and 'OverloadedLists' extensions can be used for
20+-- convenience.
21+--
22+-- 'Array' 'Char' is usually perfectly fine to buffer short to medium or even
23+-- large amounts of text in memory. Also, it is computationally efficient as
24+-- there is no encoding/decoding involved. We recommend using true streaming
25+-- operations to avoid buffering large amounts of data as long as possible e.g.
26+-- use `foldLines` instead of `lines`. However, if for some reason you are
27+-- buffering very large amounts of text in memory and are worried about space
28+-- efficiency you can use 'encodeUtf8' on the stream to convert it to a utf8
29+-- encoded 'Array' 'Word8'.
30+--
31+-- Please note the following:
32+--
33+-- * Case conversion: Some unicode characters translate to more than one code
34+-- point on case conversion. The 'toUpper' and 'toLower' functions in @base@
35+-- package do not handle such characters. Therefore, operations like @map
36+-- toUpper@ on a character stream or character array may not always perform
37+-- correct conversion.
38+-- * String comparison: In some cases, visually identical strings may have
39+-- different unicode representations, therefore, a character stream or
40+-- character array cannot be directly compared. A normalized comparison may be
41+-- needed to check string equivalence correctly.
42+
43+module Streamly.Data.Unicode.Stream
44+ (
45+ -- * Construction (Decoding)
46+ decodeChar8
47+ , decodeUtf8
48+ , decodeUtf8Lenient
49+ , D.DecodeError(..)
50+ , D.DecodeState
51+ , D.CodePoint
52+ , decodeUtf8Either
53+ , resumeDecodeUtf8Either
54+ , decodeUtf8Arrays
55+ , decodeUtf8ArraysLenient
56+
57+ -- * Elimination (Encoding)
58+ , encodeChar8
59+ , encodeChar8Unchecked
60+ , encodeUtf8
61+{-
62+ -- * Unicode aware operations
63+ , toCaseFold
64+ , toLower
65+ , toUpper
66+ , toTitle
67+
68+ -- * Operations on character strings
69+ , strip -- (dropAround isSpace)
70+ , stripEnd-}
71+ -- * Transformation
72+ , stripStart
73+ , foldLines
74+ , foldWords
75+ , unfoldLines
76+ , unfoldWords
77+
78+ -- * Streams of Strings
79+ , lines
80+ , words
81+ , unlines
82+ , unwords
83+ )
84+where
85+
86+import Streamly.Internal.Data.Unicode.Stream
87+import Prelude hiding (lines, words, unlines, unwords)
88+import qualified Streamly.Streams.StreamD as D