@@ -23,6 +23,7 @@
23module Data.Text.Internal
23module Data.Text.Internal
25 -- * Types
25 -- * Types
27 -- * Construction
28 -- * Construction
@@ -47,9 +48,9 @@ import Data.Typeable (Typeable)
48-- | A space efficient, packed, unboxed Unicode text type.
49-- | A space efficient, packed, unboxed Unicode text type.
49data Text = Text
50data Text = Text
50− {-# UNPACK #-} !A.Array -- payload
51+ {-# UNPACK #-} !A.Array -- payload (Word16 elements)
51− {-# UNPACK #-} !Int -- offset
52+ {-# UNPACK #-} !Int -- offset (units of Word16, not Char)
52− {-# UNPACK #-} !Int -- length
53+ {-# UNPACK #-} !Int -- length (units of Word16, not Char)
53 deriving (Typeable)
54 deriving (Typeable)
55-- | Smart constructor.
56-- | Smart constructor.
@@ -90,7 +91,8 @@ showText (Text arr off len) =
90-- scalar values, but are unfortunately admitted as valid 'Char'
91-- scalar values, but are unfortunately admitted as valid 'Char'
91-- values by Haskell. They cannot be represented in a 'Text'. This
92-- values by Haskell. They cannot be represented in a 'Text'. This
92-- function remaps those code points to the Unicode replacement
93-- function remaps those code points to the Unicode replacement
93−-- character \"�\", and leaves other code points unchanged.
94+-- character (U+FFFD, \'�\'), and leaves other code points
94safe :: Char -> Char
96safe :: Char -> Char
96 | ord c .&. 0x1ff800 /= 0xd800 = c
98 | ord c .&. 0x1ff800 /= 0xd800 = c
@@ -101,3 +103,22 @@ safe c
101firstf :: (a -> c) -> Maybe (a,b) -> Maybe (c,b)
103firstf :: (a -> c) -> Maybe (a,b) -> Maybe (c,b)
102firstf f (Just (a, b)) = Just (f a, b)
104firstf f (Just (a, b)) = Just (f a, b)
103firstf _ Nothing = Nothing
105firstf _ Nothing = Nothing
109+-- Internally, the 'Text' type is represented as an array of 'Word16'
110+-- UTF-16 code units. The offset and length fields in the constructor
111+-- are in these units, /not/ units of 'Char'.
113+-- Invariants that all functions must maintain:
115+-- * Since the 'Text' type uses UTF-16 internally, it cannot represent
116+-- characters in the reserved surrogate code point range U+D800 to
117+-- U+DFFF. To maintain this invariant, the 'safe' function maps
118+-- 'Char' values in this range to the replacement character (U+FFFD,
121+-- * A leading (or \"high\") surrogate code unit (0xD800–0xDBFF) must
122+-- always be followed by a trailing (or \"low\") surrogate code unit
123+-- (0xDC00-0xDFFF). A trailing surrogate code unit must always be
124+-- preceded by a leading surrogate code unit.