@@ -0,0 +1,603 @@
1+Flexible and Economical UTF-8 Decoder
2+=====================================
3+
4+Systems with elaborate Unicode support usually confront programmers with
5+a multitude of different functions and macros to process UTF-8 encoded
6+strings, often with different ideas on handling buffer boundaries, state
7+between calls, error conditions, and performance characteristics, making
8+them difficult to use correctly and efficiently. Implementations also
9+tend to be very long and complicated; one popular library has over 500
10+lines of code just for one version of the decoder. This page presents
11+one that is very easy to use correctly, short, small, fast, and free.
12+
13+Implementation in C (C99)
14+-------------------------
15+
16+ // Copyright (c) 2008-2009 Bjoern Hoehrmann <bjoern@hoehrmann.de>
17+ // See http://bjoern.hoehrmann.de/utf-8/decoder/dfa/ for details.
18+
19+ #define UTF8_ACCEPT 0
20+ #define UTF8_REJECT 1
21+
22+ static const uint8_t utf8d[] = {
23+ 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, // 00..1f
24+ 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, // 20..3f
25+ 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, // 40..5f
26+ 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, // 60..7f
27+ 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9, // 80..9f
28+ 7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7, // a0..bf
29+ 8,8,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2, // c0..df
30+ 0xa,0x3,0x3,0x3,0x3,0x3,0x3,0x3,0x3,0x3,0x3,0x3,0x3,0x4,0x3,0x3, // e0..ef
31+ 0xb,0x6,0x6,0x6,0x5,0x8,0x8,0x8,0x8,0x8,0x8,0x8,0x8,0x8,0x8,0x8, // f0..ff
32+ 0x0,0x1,0x2,0x3,0x5,0x8,0x7,0x1,0x1,0x1,0x4,0x6,0x1,0x1,0x1,0x1, // s0..s0
33+ 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,0,1,0,1,1,1,1,1,1, // s1..s2
34+ 1,2,1,1,1,1,1,2,1,2,1,1,1,1,1,1,1,1,1,1,1,1,1,2,1,1,1,1,1,1,1,1, // s3..s4
35+ 1,2,1,1,1,1,1,1,1,2,1,1,1,1,1,1,1,1,1,1,1,1,1,3,1,3,1,1,1,1,1,1, // s5..s6
36+ 1,3,1,1,1,1,1,3,1,3,1,1,1,1,1,1,1,3,1,1,1,1,1,1,1,1,1,1,1,1,1,1, // s7..s8
37+ };
38+
39+ uint32_t inline
40+ decode(uint32_t* state, uint32_t* codep, uint32_t byte) {
41+ uint32_t type = utf8d[byte];
42+
43+ *codep = (*state != UTF8_ACCEPT) ?
44+ (byte & 0x3fu) | (*codep << 6) :
45+ (0xff >> type) & (byte);
46+
47+ *state = utf8d[256 + *state*16 + type];
48+ return *state;
49+ }
50+
51+Usage
52+-----
53+
54+UTF-8 is a variable length character encoding. To decode a character one
55+or more bytes have to be read from a string. The `decode` function
56+implements a single step in this process. It takes two parameters
57+maintaining state and a byte, and returns the state achieved after
58+processing the byte. Specifically, it returns the value `UTF8_ACCEPT`
59+(0) if enough bytes have been read for a character, `UTF8_REJECT` (1) if
60+the byte is not allowed to occur at its position, and some other
61+positive value if more bytes have to be read.
62+
63+When decoding the first byte of a string, the caller must set the state
64+variable to `UTF8_ACCEPT`. If, after decoding one or more bytes the
65+state `UTF8_ACCEPT` is reached again, then the decoded Unicode character
66+value is available through the `codep` parameter. If the state
67+`UTF8_REJECT` is entered, that state will never be exited unless the
68+caller intervenes. See the examples below for more information on usage
69+and error handling, and the section on implementation details for how
70+the decoder is constructed.
71+
72+Examples
73+--------
74+
75+### Validating and counting characters
76+
77+This function checks if a null-terminated string is a well-formed UTF-8
78+sequence and counts how many code points are in the string.
79+
80+ int
81+ countCodePoints(uint8_t* s, size_t* count) {
82+ uint32_t codepoint;
83+ uint32_t state = 0;
84+
85+ for (*count = 0; *s; ++s)
86+ if (!decode(&state, &codepoint, *s))
87+ *count += 1;
88+
89+ return state != UTF8_ACCEPT;
90+ }
91+
92+It could be used like so:
93+
94+ if (countCodePoints(s, &count)) {
95+ printf("The string is malformed\n");
96+ } else {
97+ printf("The string is %u characters long\n", count);
98+ }
99+
100+### Printing code point values
101+
102+This function prints out all code points in the string and an error
103+message if unexpected bytes are encountered, or if the string ends with
104+an incomplete sequence.
105+
106+ void
107+ printCodePoints(uint8_t* s) {
108+ uint32_t codepoint;
109+ uint32_t state = 0;
110+
111+ for (; *s; ++s)
112+ if (!decode(&state, &codepoint, *s))
113+ printf("U+%04X\n", codepoint);
114+
115+ if (state != UTF8_ACCEPT)
116+ printf("The string is not well-formed\n");
117+
118+ }
119+
120+### Printing UTF-16 code units
121+
122+This loop prints out UTF-16 code units for the characters in a
123+null-terminated UTF-8 encoded string.
124+
125+ for (; *s; ++s) {
126+
127+ if (decode(&state, &codepoint, *s))
128+ continue;
129+
130+ if (codepoint <= 0xFFFF) {
131+ printf("0x%04X\n", codepoint);
132+ continue;
133+ }
134+
135+ // Encode code points above U+FFFF as surrogate pair.
136+ printf("0x%04X\n", (0xD7C0 + (codepoint >> 10)));
137+ printf("0x%04X\n", (0xDC00 + (codepoint & 0x3FF)));
138+ }
139+
140+### Error recovery
141+
142+It is sometimes desireable to recover from errors when decoding strings
143+that are supposed to be UTF-8 encoded. Programmers should be aware that
144+this can negatively affect the security properties of their application.
145+A common recovery method is to replace malformed sequences with a
146+substitute character like `U+FFFD REPLACEMENT CHARACTER`.
147+
148+Decoder implementations differ in which octets they replace and where
149+they restart. Consider for instance the sequence `0xED 0xA0 0x80`. It
150+encodes a surrogate code point which is prohibited in UTF-8. A
151+recovering decoder may replace the whole sequence and restart with the
152+next byte, or it may replace the first byte and restart with the second
153+byte, replace it, restart with the third, and replace the third byte
154+aswell.
155+
156+The following code implements one such recovery strategy. When an
157+unexpected byte is encountered, the sequence up to that point will be
158+replaced and, if the error occured in the middle of a sequence, will
159+retry the byte as if it occured at the beginning of a string. Note that
160+the decode function detects errors as early as possible, so the sequence
161+`0xED 0xA0 0x80` would result in three replacement characters.
162+
163+ for (prev = 0, current = 0; *s; prev = current, ++s) {
164+
165+ switch (decode(¤t, &codepoint, *s)) {
166+ case UTF8_ACCEPT:
167+ // A properly encoded character has been found.
168+ printf("U+%04X\n", codepoint);
169+ break;
170+
171+ case UTF8_REJECT:
172+ // The byte is invalid, replace it and restart.
173+ printf("U+FFFD (Bad UTF-8 sequence)\n");
174+ current = UTF8_ACCEPT;
175+ if (prev != UTF8_ACCEPT)
176+ s--;
177+ break;
178+ ...
179+
180+For some recovery strategies it may be useful to determine the number of
181+bytes expected. The states in the automaton are numbered such that,
182+assuming C\'s division operator, `state / 3 + 1` is that number. Of
183+course, this will only work for states other than `UTF8_ACCEPT` and
184+`UTF8_REJECT`. This number could then be used, for instance, to skip the
185+continuation octets in the illegal sequence `0xED 0xA0 0x80` so it will
186+be replaced by a single replacement character.
187+
188+### Transcoding to UTF-16 buffer
189+
190+This is a rough outline of a UTF-16 transcoder. Actual applications
191+would add code for error reporting, reporting of words written, required
192+buffer size in the case of a small buffer, and possibly other things.
193+Note that in order to avoid checking for free space in the inner loop,
194+we determine how many bytes can be read without running out of space.
195+This is one utf-8 byte per available utf-16 word, with one exception: if
196+the last byte read was the third byte in a four byte sequence we would
197+get two words for the next byte; so we read one byte less than we have
198+words available. This additional word is also needed for
199+null-termination, so it\'s never wrong to read one less.
200+
201+ int
202+ toUtf16(uint8_t* src, size_t srcBytes, uint16_t* dst, size_t dstWords, ...) {
203+
204+ uint8_t* src_actual_end = src + srcBytes;
205+ uint8_t* s = src;
206+ uint16_t* d = dst;
207+ uint32_t codepoint;
208+ uint32_t state = 0;
209+
210+ while (s < src_actual_end) {
211+
212+ size_t dst_words_free = dstWords - (d - dst);
213+ uint8_t* src_current_end = s + dst_words_free - 1;
214+
215+ if (src_actual_end < src_current_end)
216+ src_current_end = src_actual_end;
217+
218+ if (src_current_end <= s)
219+ goto toosmall;
220+
221+ while (s < src_current_end) {
222+
223+ if (decode(&state, &codepoint, *s++))
224+ continue;
225+
226+ if (codepoint > 0xffff) {
227+ *d++ = (uint16_t)(0xD7C0 + (codepoint >> 10));
228+ *d++ = (uint16_t)(0xDC00 + (codepoint & 0x3FF));
229+ } else {
230+ *d++ = (uint16_t)codepoint;
231+ }
232+ }
233+ }
234+
235+ if (state != UTF8_ACCEPT) {
236+ ...
237+ }
238+
239+ if ((dstWords - (d - dst)) == 0)
240+ goto toosmall;
241+
242+ *d++ = 0;
243+ ...
244+
245+ toosmall:
246+ ...
247+ }
248+
249+Implementation details
250+----------------------
251+
252+The `utf8d` table consists of two parts. The first part maps bytes to
253+character classes, the second part encodes a deterministic finite
254+automaton using these character classes as transitions. This section
255+details the composition of the table.
256+
257+### Canonical UTF-8 automaton
258+
259+UTF-8 is a variable length character encoding. That means state has to
260+be maintained while processing a string. The following transition graph
261+illustrates the process. We start in state zero, and whenever we come
262+back to it, we\'ve seen a whole Unicode character. Transitions not in
263+the graph are disallowed; they all lead to state one, which has been
264+omitted for readability.
265+
266+
267+
268+### Automaton with character class transitions
269+
270+The byte ranges in the transition graph above are not easily encoded in
271+the automaton in a manner that would allow fast lookup. Instead of
272+encoding the ranges directly, the ranges are split such that each byte
273+belongs to exactly one character class. Then the transitions go over
274+these character classes.
275+
276+
277+
278+### Mapping bytes to character classes
279+
280+Primarily to save space in the transition table, bytes are mapped to
281+character classes. This is the mapping:
282+
283+|||||
284+|-|-|-|-|
285+| 00..7f | 0 | 80..8f | 1 |
286+| 90..9f | 9 | a0..bf | 7 |
287+| c0..c1 | 8 | c2..df | 2 |
288+| e0..e0 | 10 | e1..ec | 3 |
289+| ed..ed | 4 | ee..ef | 3 |
290+| f0..f0 | 11 | f1..f3 | 6 |
291+| f4..f4 | 5 | f5..ff | 8 |
292+
293+
294+For bytes that may occur at the beginning of a multibyte sequence, the
295+character class number is also used to remove the most significant bits
296+from the byte, which do not contribute to the actual code point value.
297+Note that `0xc0`, `0xc1`, and `0xf5` .. `0xff` have all their bits
298+removed. These bytes cannot occur in well-formed sequences, so it does
299+not matter which bits, if any, are retained.
300+
301+|||||||||||||
302+|-|-|-|-|-|-|-|-|-|-|-|-|
303+| c0 | 8 | **11000000** | d0 | 2 | **11**010000 | e0 | 10 | **11100000** | f0 | 11 | **11110000** |
304+| c1 | 8 | **11000001** | d1 | 2 | **11**010001 | e1 | 3 | **111**00001 | f1 | 6 | **111100**01 |
305+| c2 | 2 | **11**000010 | d2 | 2 | **11**010010 | e2 | 3 | **111**00010 | f2 | 6 | **111100**10 |
306+| c3 | 2 | **11**000011 | d3 | 2 | **11**010011 | e3 | 3 | **111**00011 | f3 | 6 | **111100**11 |
307+| c4 | 2 | **11**000100 | d4 | 2 | **11**010100 | e4 | 3 | **111**00100 | f4 | 5 | **11110**100 |
308+| c5 | 2 | **11**000101 | d5 | 2 | **11**010101 | e5 | 3 | **111**00101 | f5 | 8 | **11110101** |
309+| c6 | 2 | **11**000110 | d6 | 2 | **11**010110 | e6 | 3 | **111**00110 | f6 | 8 | **11110110** |
310+| c7 | 2 | **11**000111 | d7 | 2 | **11**010111 | e7 | 3 | **111**00111 | f7 | 8 | **11110111** |
311+| c8 | 2 | **11**001000 | d8 | 2 | **11**011000 | e8 | 3 | **111**01000 | f8 | 8 | **11111000** |
312+| c9 | 2 | **11**001001 | d9 | 2 | **11**011001 | e9 | 3 | **111**01001 | f9 | 8 | **11111001** |
313+| ca | 2 | **11**001010 | da | 2 | **11**011010 | ea | 3 | **111**01010 | fa | 8 | **11111010** |
314+| cb | 2 | **11**001011 | db | 2 | **11**011011 | eb | 3 | **111**01011 | fb | 8 | **11111011** |
315+| cc | 2 | **11**001100 | dc | 2 | **11**011100 | ec | 3 | **111**01100 | fc | 8 | **11111100** |
316+| cd | 2 | **11**001101 | dd | 2 | **11**011101 | ed | 4 | **1110**1101 | fd | 8 | **11111101** |
317+| ce | 2 | **11**001110 | de | 2 | **11**011110 | ee | 3 | **111**01110 | fe | 8 | **11111110** |
318+| cf | 2 | **11**001111 | df | 2 | **11**011111 | ef | 3 | **111**01111 | ff | 8 | **11111111** |
319+
320+
321+Notes on Variations
322+-------------------
323+
324+There are several ways to change the implementation of this decoder. For
325+example, the size of the data table can be reduced, at the cost of a
326+couple more instructions, so it omits the mapping of bytes in the
327+US-ASCII range, and since all entries in the table are 4 bit values, two
328+values could be stored in a single byte.
329+
330+In some situations it may be beneficial to have a separate start state.
331+This is easily achieved by copying the s0 state in the array to the end,
332+and using the new state 9 as start state as needed.
333+
334+Where callers require the code point values, compilers tend to generate
335+slightly better code if the state calculation is moved into the
336+branches, for example
337+
338+ if (*state != UTF8_ACCEPT) {
339+ *state = utf8d[256 + *state*16 + type];
340+ *codep = (*codep << 6) | (byte & 63);
341+ } else {
342+ *state = utf8d[256 + *state*16 + type];
343+ *codep = (byte) & (255 >> type);
344+ }
345+
346+As the state will be zero in the else branch, this saves a shift and an
347+addition for each starter. Unfortunately, compilers will then typically
348+generate worse code if the codepoint value is not needed. Naturally,
349+then, two functions could be used, one that only calculates the states
350+for validation, counting, and similar applications, and one for full
351+decoding. For the sample UTF-16 transcoder a more substantial increase
352+in performance can be achieved by manually including the decode code in
353+the inner loop; then it is also worthwhile to make code points in the
354+US-ASCII range a special case:
355+
356+ while (s < src_current_end) {
357+
358+ uint32_t byte = *s++;
359+ uint32_t type = utf8d[byte];
360+
361+ if (state != UTF8_ACCEPT) {
362+ codep = (codep << 6) | (byte & 63);
363+ state = utf8d[256 + state*16 + type];
364+
365+ if (state)
366+ continue;
367+
368+ } else if (byte > 0x7f) {
369+ codep = (byte) & (255 >> type);
370+ state = utf8d[256 + type];
371+ continue;
372+
373+ } else {
374+ *d++ = (uint16_t)byte;
375+ continue;
376+ }
377+ ...
378+
379+Another variation worth of note is changing the comparison when setting
380+the code point value to this:
381+
382+ *codep = (*state > UTF8_REJECT) ?
383+ (byte & 0x3fu) | (*codep << 6) :
384+ (0xff >> type) & (byte);
385+
386+This ensures that the code point value does not exceed the value `0xff`
387+after some malformed sequence is encountered.
388+
389+As written, the decoder disallows encoding of surrogate code points,
390+overlong 2, 3, and 4 byte sequences, and 4 byte sequences outside the
391+Unicode range. Allowing them can have serious security implications, but
392+can easily be achieved by changing the character class assignments in
393+the table.
394+
395+The code samples have generally been written to perform well on my
396+system when compiled with Visual C++ 7.1 and GCC 3.4.5. Slight changes
397+may improve performance, for example, Visual C++ 7.1 will produce
398+slightly faster code when, in the manually inlined version of the
399+transcoder discussed above, the type assignment is moved into the
400+branches where it is needed, and the state and codepoint assignments in
401+the non-ASCII starter is swapped (approximately a 5% increase), but GCC
402+3.4.5 will produce considerably slower code (approximately 10%).
403+
404+I have experimented with various rearrangements of states and character
405+classes. A seemingly promising one is the following:
406+
407+
408+
409+One of the continuation ranges has been split into two, the other
410+changes are just renamings. This arrangement allows, when a continuation
411+octet is expected, to compute the character class with a shift instead
412+of a table lookup, and when looking at a non-ASCII starter, the next
413+state is simply the character class. On my system the change in
414+performance is in the area of +/- 1%. This encoding would have a number
415+of downsides: more rejecting states are required to account for
416+continuation octets where starters are expected, the table formatting
417+would use more hex notation making it longer, and calculating the number
418+of expected continuation octets from a given state is more difficult.
419+One thing I\'d still like to try out is if, perhaps by adding a couple
420+of additional states, for continuation states the next state can be
421+computed without any table lookup with a few easily paired instructions.
422+
423+On 24th June 2010 Rich Felker pointed out that the state values in the
424+transition table can be pre-multiplied with 16 which would save a shift
425+instruction for every byte. D\'oh! We actually just need 12 and can
426+throw away the filler values previously in the table making the table 36
427+bytes shorter and save the shift in the code.
428+
429+ // Copyright (c) 2008-2010 Bjoern Hoehrmann <bjoern@hoehrmann.de>
430+ // See http://bjoern.hoehrmann.de/utf-8/decoder/dfa/ for details.
431+
432+ #define UTF8_ACCEPT 0
433+ #define UTF8_REJECT 12
434+
435+ static const uint8_t utf8d[] = {
436+ // The first part of the table maps bytes to character classes that
437+ // to reduce the size of the transition table and create bitmasks.
438+ 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
439+ 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
440+ 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
441+ 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
442+ 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, 9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,
443+ 7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7, 7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,
444+ 8,8,2,2,2,2,2,2,2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,
445+ 10,3,3,3,3,3,3,3,3,3,3,3,3,4,3,3, 11,6,6,6,5,8,8,8,8,8,8,8,8,8,8,8,
446+
447+ // The second part is a transition table that maps a combination
448+ // of a state of the automaton and a character class to a state.
449+ 0,12,24,36,60,96,84,12,12,12,48,72, 12,12,12,12,12,12,12,12,12,12,12,12,
450+ 12, 0,12,12,12,12,12, 0,12, 0,12,12, 12,24,12,12,12,12,12,24,12,24,12,12,
451+ 12,12,12,12,12,12,12,24,12,12,12,12, 12,24,12,12,12,12,12,12,12,24,12,12,
452+ 12,12,12,12,12,12,12,36,12,36,12,12, 12,36,12,12,12,12,12,36,12,36,12,12,
453+ 12,36,12,12,12,12,12,12,12,12,12,12,
454+ };
455+
456+ uint32_t inline
457+ decode(uint32_t* state, uint32_t* codep, uint32_t byte) {
458+ uint32_t type = utf8d[byte];
459+
460+ *codep = (*state != UTF8_ACCEPT) ?
461+ (byte & 0x3fu) | (*codep << 6) :
462+ (0xff >> type) & (byte);
463+
464+ *state = utf8d[256 + *state + type];
465+ return *state;
466+ }
467+
468+Notes on performance
469+--------------------
470+
471+To conduct some ad-hoc performance testing I\'ve used three different
472+UTF-8 encoded buffers and passed them through a couple of UTF-8 to
473+UTF-16 transcoders. The large buffer is a April 2009 Hindi Wikipedia
474+article XML dump, the medium buffer Markus Kuhn\'s UTF-8-demo.txt, and
475+the tiny buffer my name, each about the number of times required for
476+about 1GB of data. All tests ran on a [Intel Prescott
477+Celeron](http://en.wikipedia.org/wiki/Celeron#Prescott-256) at 2666 MHz.
478+See [Changes](#changes) for some additional details.
479+
480+ | | Large | Medium | Tiny |
481+ |---------------------------------------------------------------------------|---------| ---------|----------|
482+ |`NS_CStringToUTF16()` Mozilla 1.9 (*includes malloc/free time*) | 36924ms| 39773ms| 107958ms|
483+ |`iconv()` 1.9 compiled with Visual C++ (Cygwin iconv 1.11 similar) | 22740ms| 21765ms| 32595ms|
484+ |`g_utf8_to_utf16()` Cygwin Glib 2.0 (*includes malloc/free time*) | 21599ms| 20345ms| 98782ms|
485+ |`ConvertUTF8toUTF16()` Unicode Inc., Visual C++ 7.1 -Ox -Ot -G7 | 11183ms| 11251ms| 19453ms|
486+ |`MultiByteToWideChar()` Windows API (Server 2003 SP2) | 9857ms| 10779ms| 12771ms|
487+ |`u_strFromUTF8` from ICU 4.0.1 (Visual Studio 2008, web site distribution) | 8778ms| 5223ms| 5419ms|
488+ |`PyUnicode_DecodeUTF8Stateful` (3.1a2), Visual C++ 7.1 -Ox -Ot -G7 | 4523ms| 5686ms| 3138ms|
489+ |Example section transcoder, Visual C++ 7.1 -Ox -Ot -G7 | 5397ms| 5789ms| 6250ms|
490+ |Manually inlined transcoder (see above), Visual C++ 7.1 -Ox -Ot -G7 | 4277ms| 4998ms| 4640ms|
491+ |Same, Cygwin GCC 3.4.5 -march=prescott -fomit-frame-pointer -O3 | 4492ms| 5154ms| 4432ms|
492+ |Same, Cygwin GCC 4.3.2 -march=prescott -fomit-frame-pointer -O3 | 5439ms| 6322ms| 5567ms|
493+ |Same, Visual C++ 6.0 -TP -O2 | 5398ms| 6259ms| 6446ms|
494+ |Same, Visual C++ 7.1 -Ox -Ot -G7 (*includes malloc/free time*) | 5498ms| 5086ms| 25852ms|
495+
496+I have also timed functions that `xor` all code points in the large
497+buffer. In Visual Studio 2008 ICU\'s `U8_NEXT` macro comes out at
498+\~8000ms, the `U8_NEXT_UNSAFE` macro, which requires complete and
499+well-formed input, at \~4000ms, and the `decode` function is at
500+\~5900ms. Using the same manual inlining as for the transcode function,
501+Cygwin GCC 3.4.5 -march=prescott -O3 -fomit-frame-pointer brings it down
502+to roughly the same times as the transcode function for all three
503+buffers.
504+
505+While these results do not model real-world applications well, it seems
506+reasonable to suggest that the reduced complexity does not come at the
507+price of reduced performance. Note that instructions that compute the
508+code point values will generally be optimized away when not needed. For
509+example, checking if a null-terminated string is properly UTF-8 encoded
510+\...
511+
512+ int
513+ IsUTF8(uint8_t* s) {
514+ uint32_t codepoint, state = 0;
515+
516+ while (*s)
517+ decode(&state, &codepoint, *s++);
518+
519+ return state == UTF8_ACCEPT;
520+ }
521+
522+\... does not require the individual code point values, and so the loop
523+becomes something like this:
524+
525+ l1: movzx eax,al
526+ shl edx,4
527+ add ecx,1
528+ movzx eax,byte ptr [eax+404000h]
529+ movzx edx,byte ptr [eax+edx+256+404000h]
530+ movzx eax,byte ptr [ecx]
531+ test al,al
532+ jne l1
533+
534+For comparison, this is a typical `strlen` loop:
535+
536+ l1: mov cl,byte ptr [eax]
537+ add eax,1
538+ test cl,cl
539+ jne l1
540+
541+With the large buffer and the same number of times as above, `strlen`
542+takes 1507ms while `IsUTF8` takes 2514ms.
543+
544+License
545+-------
546+
547+Copyright (c) 2008-2009 [Bjoern Hoehrmann](http://bjoern.hoehrmann.de/)
548+\<<bjoern@hoehrmann.de>\>
549+
550+Permission is hereby granted, free of charge, to any person obtaining a
551+copy of this software and associated documentation files (the
552+\"Software\"), to deal in the Software without restriction, including
553+without limitation the rights to use, copy, modify, merge, publish,
554+distribute, sublicense, and/or sell copies of the Software, and to
555+permit persons to whom the Software is furnished to do so, subject to
556+the following conditions:
557+
558+The above copyright notice and this permission notice shall be included
559+in all copies or substantial portions of the Software.
560+
561+THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND,
562+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
563+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
564+IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
565+CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
566+TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
567+SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
568+:::
569+
570+Changes
571+-------
572+
573+25 Jun 2010
574+: Added an improved variation based on an observation from Rich
575+ Felker.
576+
577+30 April 2009
578+: Added some more items to the performance table: the manually inlined
579+ transcoder allocating worst case memory for each run and freeing it
580+ before the next run; and results for Mozilla\'s NS\_CStringToUTF16
581+ (a new nsAutoString is created for each run, and truncated before
582+ the next). This used the XULRunner SDK 1.9.0.7 binary distribution
583+ from the Mozilla website.
584+
585+18 April 2009
586+: Added notes to the Variations section on handling malformed
587+ sequences and failed optimization attempts.
588+
589+14 April 2009
590+: Added PyUnicode\_DecodeUTF8Stateful times; the function has been
591+ modified slightly so it works outside Python and so it uses a
592+ pre-allocated buffer. Normally does not check output buffer
593+ boundaries but rather allocates a worst case buffer, then resizes
594+ it. Apparently the decoder [allows encodings of surrogate code
595+ points](http://bugs.python.org/issue3672).
596+
597+Author
598+------
599+
600+[Björn Höhrmann](http://bjoern.hoehrmann.de) <bjoern@hoehrmann.de>
601+([Donate via
602+SourceForge](http://sourceforge.net/developer/user_donations.php?user_id=188003),
603+[PayPal](https://www.paypal.com/cgi-bin/webscr?cmd=_xclick&business=bjoern@hoehrmann.de&item_name=Support+Bjoern+Hoehrmann))