c91deda · text · git.iko.soy

git.iko.soy/text

Be more precise about meaning of "character"

commitc91dedad7fd35bf7d76fc7dced5aff66310e9d97 authorBen Gamari <ben@smart-cactus.org> dateMon 9 Oct 2017, 17:11 (9 years ago) parentcda0e3f

2 files changed+19−4

compare with master

unifiedsplit

Data/Text.hs+15 Data/Text/Internal/Fusion/Types.hs+4−4

Data/Text.hs+15

@@ -34,6 +34,7 @@

3434-- with Unicode text (including normalization, regular expressions,

3535-- non-standard encodings, text breaking, and locales), see

3636-- <http://hackage.haskell.org/package/text-icu the text-icu package >.

37+--

3738

3839module Data.Text

3940 (

@@ -43,6 +44,9 @@ module Data.Text

4344 -- * Acceptable data

4445 -- $replacement

4546

47+ -- * Definition of character

48+ -- $character_definition

49+

4650 -- * Fusion

4751 -- $fusion

4852

@@ -247,6 +251,17 @@ import qualified GHC.Exts as Exts

247251import Text.Printf (PrintfArg, formatArg, formatString)

248252#endif

249253

254+-- $character-definition

255+--

256+-- This package uses the word /character/ to denote Unicode code points.

257+--

258+-- Note that this is not the same thing as a grapheme (e.g. a composition of

259+-- code points that form one visual symbol). For instance, consider the grapheme

260+-- @ä@. This symbol has two Unicode representations: a single code-point

261+-- representation @U+00E4@ (the @LATIN SMALL LETTER A WITH DIAERESIS@ code

262+-- point), and a two code point representation @U+0061@ (the @A@ code point) and

263+-- @U+0308@ (the @COMBINING DIAERESIS@ code point).

264+

250265-- $strict

251266--

252267-- This package provides both strict and lazy 'Text' types. The

Data/Text/Internal/Fusion/Types.hs+4−4

@@ -75,16 +75,16 @@ instance (Ord a) => Ord (Stream a) where

7575-- unstreaming functions must be able to cope with the hint being too

7676-- small or too large.

7777--

78−-- The size hint tries to track the UTF-16 code points in a stream,

79−-- but often counts the number of characters instead. It can easily

78+-- The size hint tries to track the UTF-16 code units in a stream,

79+-- but often counts the number of code points instead. It can easily

8080-- undercount if, for instance, a transformed stream contains astral

81−-- plane characters (those above 0x10000).

81+-- plane code points (those above 0x10000).

8282

8383data Stream a =

8484 forall s. Stream

8585 (s -> Step s a) -- stepper function

8686 !s -- current state

87− !Size -- size hint

87+ !Size -- size hint in code units

8888

8989-- | /O(n)/ Determines if two streams are equal.

9090eq :: (Eq a) => Stream a -> Stream a -> Bool