git.iko.soy/text
on master help sign in
Be more precise about meaning of "character"
c91dedaBen Gamari
Data/Text.hs+15
compare with master
unifiedsplit
@@ -34,6 +34,7 @@
34-- with Unicode text (including normalization, regular expressions,
34-- with Unicode text (including normalization, regular expressions,
35-- non-standard encodings, text breaking, and locales), see
35-- non-standard encodings, text breaking, and locales), see
36-- <http://hackage.haskell.org/package/text-icu the text-icu package >.
36-- <http://hackage.haskell.org/package/text-icu the text-icu package >.
37+--
37
38
38module Data.Text
39module Data.Text
39 (
40 (
@@ -43,6 +44,9 @@ module Data.Text
43 -- * Acceptable data
44 -- * Acceptable data
44 -- $replacement
45 -- $replacement
45
46
47+ -- * Definition of character
48+ -- $character_definition
49+
46 -- * Fusion
50 -- * Fusion
47 -- $fusion
51 -- $fusion
48
52
@@ -247,6 +251,17 @@ import qualified GHC.Exts as Exts
247import Text.Printf (PrintfArg, formatArg, formatString)
251import Text.Printf (PrintfArg, formatArg, formatString)
248#endif
252#endif
249
253
254+-- $character-definition
255+--
256+-- This package uses the word /character/ to denote Unicode code points.
257+--
258+-- Note that this is not the same thing as a grapheme (e.g. a composition of
259+-- code points that form one visual symbol). For instance, consider the grapheme
260+-- @ä@. This symbol has two Unicode representations: a single code-point
261+-- representation @U+00E4@ (the @LATIN SMALL LETTER A WITH DIAERESIS@ code
262+-- point), and a two code point representation @U+0061@ (the @A@ code point) and
263+-- @U+0308@ (the @COMBINING DIAERESIS@ code point).
264+
250-- $strict
265-- $strict
251--
266--
252-- This package provides both strict and lazy 'Text' types. The
267-- This package provides both strict and lazy 'Text' types. The
Data/Text/Internal/Fusion/Types.hs+4−4
@@ -75,16 +75,16 @@ instance (Ord a) => Ord (Stream a) where
75-- unstreaming functions must be able to cope with the hint being too
75-- unstreaming functions must be able to cope with the hint being too
76-- small or too large.
76-- small or too large.
77--
77--
78−-- The size hint tries to track the UTF-16 code points in a stream,
78+-- The size hint tries to track the UTF-16 code units in a stream,
79−-- but often counts the number of characters instead. It can easily
79+-- but often counts the number of code points instead. It can easily
80-- undercount if, for instance, a transformed stream contains astral
80-- undercount if, for instance, a transformed stream contains astral
81−-- plane characters (those above 0x10000).
81+-- plane code points (those above 0x10000).
82
82
83data Stream a =
83data Stream a =
84 forall s. Stream
84 forall s. Stream
85 (s -> Step s a) -- stepper function
85 (s -> Step s a) -- stepper function
86 !s -- current state
86 !s -- current state
87− !Size -- size hint
87+ !Size -- size hint in code units
88
88
89-- | /O(n)/ Determines if two streams are equal.
89-- | /O(n)/ Determines if two streams are equal.
90eq :: (Eq a) => Stream a -> Stream a -> Bool
90eq :: (Eq a) => Stream a -> Stream a -> Bool