@@ -23,6 +23,7 @@
2323module Data.Text.Internal
2424 (
2525 -- * Types
26+ -- $internals
2627 Text(..)
2728 -- * Construction
2829 , text
@@ -47,9 +48,9 @@ import Data.Typeable (Typeable)
4748
4849-- | A space efficient, packed, unboxed Unicode text type.
4950data Text = Text
50− {-# UNPACK #-} !A.Array -- payload
51− {-# UNPACK #-} !Int -- offset
52− {-# UNPACK #-} !Int -- length
51+ {-# UNPACK #-} !A.Array -- payload (Word16 elements)
52+ {-# UNPACK #-} !Int -- offset (units of Word16, not Char)
53+ {-# UNPACK #-} !Int -- length (units of Word16, not Char)
5354 deriving (Typeable)
5455
5556-- | Smart constructor.
@@ -90,7 +91,8 @@ showText (Text arr off len) =
9091-- scalar values, but are unfortunately admitted as valid 'Char'
9192-- values by Haskell. They cannot be represented in a 'Text'. This
9293-- function remaps those code points to the Unicode replacement
93−-- character \"�\", and leaves other code points unchanged.
94+-- character (U+FFFD, \'�\'), and leaves other code points
95+-- unchanged.
9496safe :: Char -> Char
9597safe c
9698 | ord c .&. 0x1ff800 /= 0xd800 = c
@@ -101,3 +103,22 @@ safe c
101103firstf :: (a -> c) -> Maybe (a,b) -> Maybe (c,b)
102104firstf f (Just (a, b)) = Just (f a, b)
103105firstf _ Nothing = Nothing
106+
107+-- $internals
108+--
109+-- Internally, the 'Text' type is represented as an array of 'Word16'
110+-- UTF-16 code units. The offset and length fields in the constructor
111+-- are in these units, /not/ units of 'Char'.
112+--
113+-- Invariants that all functions must maintain:
114+--
115+-- * Since the 'Text' type uses UTF-16 internally, it cannot represent
116+-- characters in the reserved surrogate code point range U+D800 to
117+-- U+DFFF. To maintain this invariant, the 'safe' function maps
118+-- 'Char' values in this range to the replacement character (U+FFFD,
119+-- \'�\').
120+--
121+-- * A leading (or \"high\") surrogate code unit (0xD800–0xDBFF) must
122+-- always be followed by a trailing (or \"low\") surrogate code unit
123+-- (0xDC00-0xDFFF). A trailing surrogate code unit must always be
124+-- preceded by a leading surrogate code unit.