git.iko.soy/text
on 3cf8564 help sign in
text/Data/Text/Encoding/Utf8.hs
161 lines 4.6 kB history raw
1{-# LANGUAGE CPP, MagicHash, BangPatterns #-}
3-- |
4-- Module : Data.Text.Encoding.Utf8
5-- Copyright : (c) 2008, 2009 Tom Harper,
6-- (c) 2009, 2010 Bryan O'Sullivan,
7-- (c) 2009 Duncan Coutts
8--
9-- License : BSD-style
10-- Maintainer : bos@serpentine.com, rtomharper@googlemail.com,
11-- duncan@haskell.org
12-- Stability : experimental
13-- Portability : GHC
14--
15-- Basic UTF-8 validation and character manipulation.
16module Data.Text.Encoding.Utf8
17 (
18 -- Decomposition
19 ord2
20 , ord3
21 , ord4
22 -- Construction
23 , chr2
24 , chr3
25 , chr4
26 -- * Validation
27 , validate1
28 , validate2
29 , validate3
30 , validate4
31 ) where
33#if defined(ASSERTS)
34import Control.Exception (assert)
35#endif
36import Data.Bits ((.&.))
37import Data.Text.UnsafeChar (ord)
38import Data.Text.UnsafeShift (shiftR)
39import GHC.Exts
40import GHC.Word (Word8(..))
42default(Int)
44between :: Word8 -- ^ byte to check
45 -> Word8 -- ^ lower bound
46 -> Word8 -- ^ upper bound
47 -> Bool
48between x y z = x >= y && x <= z
49{-# INLINE between #-}
51ord2 :: Char -> (Word8,Word8)
52ord2 c =
53#if defined(ASSERTS)
54 assert (n >= 0x80 && n <= 0x07ff)
55#endif
56 (x1,x2)
57 where
58 n = ord c
59 x1 = fromIntegral $ (n `shiftR` 6) + 0xC0
60 x2 = fromIntegral $ (n .&. 0x3F) + 0x80
62ord3 :: Char -> (Word8,Word8,Word8)
63ord3 c =
64#if defined(ASSERTS)
65 assert (n >= 0x0800 && n <= 0xffff)
66#endif
67 (x1,x2,x3)
68 where
69 n = ord c
70 x1 = fromIntegral $ (n `shiftR` 12) + 0xE0
71 x2 = fromIntegral $ ((n `shiftR` 6) .&. 0x3F) + 0x80
72 x3 = fromIntegral $ (n .&. 0x3F) + 0x80
74ord4 :: Char -> (Word8,Word8,Word8,Word8)
75ord4 c =
76#if defined(ASSERTS)
77 assert (n >= 0x10000)
78#endif
79 (x1,x2,x3,x4)
80 where
81 n = ord c
82 x1 = fromIntegral $ (n `shiftR` 18) + 0xF0
83 x2 = fromIntegral $ ((n `shiftR` 12) .&. 0x3F) + 0x80
84 x3 = fromIntegral $ ((n `shiftR` 6) .&. 0x3F) + 0x80
85 x4 = fromIntegral $ (n .&. 0x3F) + 0x80
87chr2 :: Word8 -> Word8 -> Char
88chr2 (W8# x1#) (W8# x2#) = C# (chr# (z1# +# z2#))
89 where
90 !y1# = word2Int# x1#
91 !y2# = word2Int# x2#
92 !z1# = uncheckedIShiftL# (y1# -# 0xC0#) 6#
93 !z2# = y2# -# 0x80#
94{-# INLINE chr2 #-}
96chr3 :: Word8 -> Word8 -> Word8 -> Char
97chr3 (W8# x1#) (W8# x2#) (W8# x3#) = C# (chr# (z1# +# z2# +# z3#))
98 where
99 !y1# = word2Int# x1#
100 !y2# = word2Int# x2#
101 !y3# = word2Int# x3#
102 !z1# = uncheckedIShiftL# (y1# -# 0xE0#) 12#
103 !z2# = uncheckedIShiftL# (y2# -# 0x80#) 6#
104 !z3# = y3# -# 0x80#
105{-# INLINE chr3 #-}
107chr4 :: Word8 -> Word8 -> Word8 -> Word8 -> Char
108chr4 (W8# x1#) (W8# x2#) (W8# x3#) (W8# x4#) =
109 C# (chr# (z1# +# z2# +# z3# +# z4#))
110 where
111 !y1# = word2Int# x1#
112 !y2# = word2Int# x2#
113 !y3# = word2Int# x3#
114 !y4# = word2Int# x4#
115 !z1# = uncheckedIShiftL# (y1# -# 0xF0#) 18#
116 !z2# = uncheckedIShiftL# (y2# -# 0x80#) 12#
117 !z3# = uncheckedIShiftL# (y3# -# 0x80#) 6#
118 !z4# = y4# -# 0x80#
119{-# INLINE chr4 #-}
121validate1 :: Word8 -> Bool
122validate1 x1 = x1 <= 0x7F
123{-# INLINE validate1 #-}
125validate2 :: Word8 -> Word8 -> Bool
126validate2 x1 x2 = between x1 0xC2 0xDF && between x2 0x80 0xBF
127{-# INLINE validate2 #-}
129validate3 :: Word8 -> Word8 -> Word8 -> Bool
130{-# INLINE validate3 #-}
131validate3 x1 x2 x3 = validate3_1 || validate3_2 || validate3_3 || validate3_4
132 where
133 validate3_1 = (x1 == 0xE0) &&
134 between x2 0xA0 0xBF &&
135 between x3 0x80 0xBF
136 validate3_2 = between x1 0xE1 0xEC &&
137 between x2 0x80 0xBF &&
138 between x3 0x80 0xBF
139 validate3_3 = x1 == 0xED &&
140 between x2 0x80 0x9F &&
141 between x3 0x80 0xBF
142 validate3_4 = between x1 0xEE 0xEF &&
143 between x2 0x80 0xBF &&
144 between x3 0x80 0xBF
146validate4 :: Word8 -> Word8 -> Word8 -> Word8 -> Bool
147{-# INLINE validate4 #-}
148validate4 x1 x2 x3 x4 = validate4_1 || validate4_2 || validate4_3
149 where
150 validate4_1 = x1 == 0xF0 &&
151 between x2 0x90 0xBF &&
152 between x3 0x80 0xBF &&
153 between x4 0x80 0xBF
154 validate4_2 = between x1 0xF1 0xF3 &&
155 between x2 0x80 0xBF &&
156 between x3 0x80 0xBF &&
157 between x4 0x80 0xBF
158 validate4_3 = x1 == 0xF4 &&
159 between x2 0x80 0x8F &&
160 between x3 0x80 0xBF &&
161 between x4 0x80 0xBF