git.iko.soy/streamly
on ecb846d help sign in
streamly/src/Streamly/Internal/Data/Unicode/Stream.hs
230 lines 6.8 kB history raw
1{-# OPTIONS_HADDOCK hide #-}
2{-# LANGUAGE FlexibleContexts #-}
4-- |
5-- Module : Streamly.Data.Internal.Unicode.Stream
6-- Copyright : (c) 2018 Composewell Technologies
7--
8-- License : BSD3
9-- Maintainer : streamly@composewell.com
10-- Stability : experimental
11-- Portability : GHC
12--
13module Streamly.Internal.Data.Unicode.Stream
14 (
15 -- * Construction (Decoding)
16 decodeLatin1
17 , decodeUtf8
18 , decodeUtf8Lax
19 , D.DecodeError(..)
20 , D.DecodeState
21 , D.CodePoint
22 , decodeUtf8Either
23 , resumeDecodeUtf8Either
24 , decodeUtf8Arrays
25 , decodeUtf8ArraysLenient
27 -- * Elimination (Encoding)
28 , encodeLatin1
29 , encodeLatin1Lax
30 , encodeUtf8
31 {-
32 -- * Operations on character strings
33 , strip -- (dropAround isSpace)
34 , stripEnd
35 -}
36 -- * Transformation
37 , stripStart
38 , lines
39 , words
40 , unlines
41 , unwords
42 )
43where
45import Control.Monad.IO.Class (MonadIO)
46import Data.Char (ord)
47import Data.Word (Word8)
48import GHC.Base (unsafeChr)
49import Streamly (IsStream)
50import Prelude hiding (String, lines, words, unlines, unwords)
51import Streamly.Data.Fold (Fold)
52import Streamly.Memory.Array (Array)
53import Streamly.Internal.Data.Unfold (Unfold)
55import qualified Streamly.Internal.Prelude as S
56import qualified Streamly.Streams.StreamD as D
58-------------------------------------------------------------------------------
59-- Encoding/Decoding Unicode Characters
60-------------------------------------------------------------------------------
62-- | Decode a stream of bytes to Unicode characters by mapping each byte to a
63-- corresponding Unicode 'Char' in 0-255 range.
64--
65-- /Since: 0.7.0/
66{-# INLINE decodeLatin1 #-}
67decodeLatin1 :: (IsStream t, Monad m) => t m Word8 -> t m Char
68decodeLatin1 = S.map (unsafeChr . fromIntegral)
70-- | Encode a stream of Unicode characters to bytes by mapping each character
71-- to a byte in 0-255 range. Throws an error if the input stream contains
72-- characters beyond 255.
73--
74-- /Since: 0.7.0/
75{-# INLINE encodeLatin1 #-}
76encodeLatin1 :: (IsStream t, Monad m) => t m Char -> t m Word8
77encodeLatin1 = S.map convert
78 where
79 convert c =
80 let codepoint = ord c
81 in if codepoint > 255
82 then error $ "Streamly.String.encodeLatin1 invalid \
83 \input char codepoint " ++ show codepoint
84 else fromIntegral codepoint
86-- | Like 'encodeLatin1' but silently truncates and maps input characters beyond
87-- 255 to (incorrect) chars in 0-255 range. No error or exception is thrown
88-- when such truncation occurs.
89--
90-- /Since: 0.7.0/
91{-# INLINE encodeLatin1Lax #-}
92encodeLatin1Lax :: (IsStream t, Monad m) => t m Char -> t m Word8
93encodeLatin1Lax = S.map (fromIntegral . ord)
95-- | Decode a UTF-8 encoded bytestream to a stream of Unicode characters.
96-- The incoming stream is truncated if an invalid codepoint is encountered.
97--
98-- /Since: 0.7.0/
99{-# INLINE decodeUtf8 #-}
100decodeUtf8 :: (Monad m, IsStream t) => t m Word8 -> t m Char
101decodeUtf8 = D.fromStreamD . D.decodeUtf8 . D.toStreamD
103-- |
104--
105-- /Internal/
106{-# INLINE decodeUtf8Arrays #-}
107decodeUtf8Arrays :: (MonadIO m, IsStream t) => t m (Array Word8) -> t m Char
108decodeUtf8Arrays = D.fromStreamD . D.decodeUtf8Arrays . D.toStreamD
110-- | Decode a UTF-8 encoded bytestream to a stream of Unicode characters.
111-- Any invalid codepoint encountered is replaced with the unicode replacement
112-- character.
113--
114-- /Since: 0.7.0/
115{-# INLINE decodeUtf8Lax #-}
116decodeUtf8Lax :: (Monad m, IsStream t) => t m Word8 -> t m Char
117decodeUtf8Lax = D.fromStreamD . D.decodeUtf8Lenient . D.toStreamD
119-- |
120--
121-- /Internal/
122{-# INLINE decodeUtf8Either #-}
123decodeUtf8Either :: (Monad m, IsStream t)
124 => t m Word8 -> t m (Either D.DecodeError Char)
125decodeUtf8Either = D.fromStreamD . D.decodeUtf8Either . D.toStreamD
127-- |
128--
129-- /Internal/
130{-# INLINE resumeDecodeUtf8Either #-}
131resumeDecodeUtf8Either
132 :: (Monad m, IsStream t)
133 => D.DecodeState
134 -> D.CodePoint
135 -> t m Word8
136 -> t m (Either D.DecodeError Char)
137resumeDecodeUtf8Either st cp =
138 D.fromStreamD . D.resumeDecodeUtf8Either st cp . D.toStreamD
140-- |
141--
142-- /Internal/
143{-# INLINE decodeUtf8ArraysLenient #-}
144decodeUtf8ArraysLenient ::
145 (MonadIO m, IsStream t) => t m (Array Word8) -> t m Char
146decodeUtf8ArraysLenient =
147 D.fromStreamD . D.decodeUtf8ArraysLenient . D.toStreamD
149-- | Encode a stream of Unicode characters to a UTF-8 encoded bytestream.
150--
151-- /Since: 0.7.0/
152{-# INLINE encodeUtf8 #-}
153encodeUtf8 :: (Monad m, IsStream t) => t m Char -> t m Word8
154encodeUtf8 = D.fromStreamD . D.encodeUtf8 . D.toStreamD
156{-
157-------------------------------------------------------------------------------
158-- Utility operations on strings
159-------------------------------------------------------------------------------
161strip :: IsStream t => t m Char -> t m Char
162strip = undefined
164stripEnd :: IsStream t => t m Char -> t m Char
165stripEnd = undefined
166-}
168-- | Remove leading whitespace from a string.
169--
170-- > stripStart = S.dropWhile isSpace
171--
172-- /Internal/
173{-# INLINE stripStart #-}
174stripStart :: (Monad m, IsStream t) => t m Char -> t m Char
175stripStart = S.dropWhile isSpace
177-- | Fold each line of the stream using the supplied 'Fold'
178-- and stream the result.
179--
180-- >>> S.toList $ lines FL.toList (S.fromList "lines\nthis\nstring\n\n\n")
181-- ["lines", "this", "string", "", ""]
182--
183-- > lines = S.splitOnSuffix (== '\n')
184--
185-- /Internal/
186{-# INLINE lines #-}
187lines :: (Monad m, IsStream t) => Fold m Char b -> t m Char -> t m b
188lines = S.splitOnSuffix (== '\n')
190foreign import ccall unsafe "u_iswspace"
191 iswspace :: Int -> Int
193-- | Code copied from base/Data.Char to INLINE it
194{-# INLINE isSpace #-}
195isSpace :: Char -> Bool
196isSpace c
197 | uc <= 0x377 = uc == 32 || uc - 0x9 <= 4 || uc == 0xa0
198 | otherwise = iswspace (ord c) /= 0
199 where
200 uc = fromIntegral (ord c) :: Word
202-- | Fold each word of the stream using the supplied 'Fold'
203-- and stream the result.
204--
205-- >>> S.toList $ words FL.toList (S.fromList "fold these words")
206-- ["fold", "these", "words"]
207--
208-- > words = S.wordsBy isSpace
209--
210-- /Internal/
211{-# INLINE words #-}
212words :: (Monad m, IsStream t) => Fold m Char b -> t m Char -> t m b
213words = S.wordsBy isSpace
215-- | Unfold a stream to character streams using the supplied 'Unfold'
216-- and concat the results suffixing a newline character @\\n@ to each stream.
217--
218-- /Internal/
219{-# INLINE unlines #-}
220unlines :: (MonadIO m, IsStream t) => Unfold m a Char -> t m a -> t m Char
221unlines = S.interposeSuffix '\n'
223-- | Unfold the elements of a stream to character streams using the supplied
224-- 'Unfold' and concat the results with a whitespace character infixed between
225-- the streams.
226--
227-- /Internal/
228{-# INLINE unwords #-}
229unwords :: (MonadIO m, IsStream t) => Unfold m a Char -> t m a -> t m Char
230unwords = S.interpose ' '