|
1
|
1 // utf.c
|
|
|
2 // Copyright (c) 2003 by Digital Mars
|
|
|
3 // All Rights Reserved
|
|
|
4 // written by Walter Bright
|
|
|
5 // http://www.digitalmars.com
|
|
|
6 // License for redistribution is by either the Artistic License
|
|
|
7 // in artistic.txt, or the GNU General Public License in gnu.txt.
|
|
|
8 // See the included readme.txt for details.
|
|
|
9
|
|
|
10 // Description of UTF-8 at:
|
|
|
11 // http://www.cl.cam.ac.uk/~mgk25/unicode.html#utf-8
|
|
|
12
|
|
|
13 #include <stdio.h>
|
|
|
14 #include <assert.h>
|
|
|
15
|
|
|
16 #include "utf.h"
|
|
|
17
|
|
|
18 int utf_isValidDchar(dchar_t c)
|
|
|
19 {
|
|
|
20 return c < 0xD800 ||
|
|
|
21 (c > 0xDFFF && c <= 0x10FFFF && c != 0xFFFE && c != 0xFFFF);
|
|
|
22 }
|
|
|
23
|
|
|
24 /********************************************
|
|
|
25 * Decode a single UTF-8 character sequence.
|
|
|
26 * Returns:
|
|
|
27 * NULL success
|
|
|
28 * !=NULL error message string
|
|
|
29 */
|
|
|
30
|
|
|
31 char *utf_decodeChar(unsigned char *s, size_t len, size_t *pidx, dchar_t *presult)
|
|
|
32 {
|
|
|
33 dchar_t V;
|
|
|
34 size_t i = *pidx;
|
|
|
35 unsigned char u = s[i];
|
|
|
36
|
|
|
37 assert(i >= 0 && i < len);
|
|
|
38
|
|
|
39 if (u & 0x80)
|
|
|
40 { unsigned n;
|
|
|
41 unsigned char u2;
|
|
|
42
|
|
|
43 /* The following encodings are valid, except for the 5 and 6 byte
|
|
|
44 * combinations:
|
|
|
45 * 0xxxxxxx
|
|
|
46 * 110xxxxx 10xxxxxx
|
|
|
47 * 1110xxxx 10xxxxxx 10xxxxxx
|
|
|
48 * 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx
|
|
|
49 * 111110xx 10xxxxxx 10xxxxxx 10xxxxxx 10xxxxxx
|
|
|
50 * 1111110x 10xxxxxx 10xxxxxx 10xxxxxx 10xxxxxx 10xxxxxx
|
|
|
51 */
|
|
|
52 for (n = 1; ; n++)
|
|
|
53 {
|
|
|
54 if (n > 4)
|
|
|
55 goto Lerr; // only do the first 4 of 6 encodings
|
|
|
56 if (((u << n) & 0x80) == 0)
|
|
|
57 {
|
|
|
58 if (n == 1)
|
|
|
59 goto Lerr;
|
|
|
60 break;
|
|
|
61 }
|
|
|
62 }
|
|
|
63
|
|
|
64 // Pick off (7 - n) significant bits of B from first byte of octet
|
|
|
65 V = (dchar_t)(u & ((1 << (7 - n)) - 1));
|
|
|
66
|
|
|
67 if (i + (n - 1) >= len)
|
|
|
68 goto Lerr; // off end of string
|
|
|
69
|
|
|
70 /* The following combinations are overlong, and illegal:
|
|
|
71 * 1100000x (10xxxxxx)
|
|
|
72 * 11100000 100xxxxx (10xxxxxx)
|
|
|
73 * 11110000 1000xxxx (10xxxxxx 10xxxxxx)
|
|
|
74 * 11111000 10000xxx (10xxxxxx 10xxxxxx 10xxxxxx)
|
|
|
75 * 11111100 100000xx (10xxxxxx 10xxxxxx 10xxxxxx 10xxxxxx)
|
|
|
76 */
|
|
|
77 u2 = s[i + 1];
|
|
|
78 if ((u & 0xFE) == 0xC0 ||
|
|
|
79 (u == 0xE0 && (u2 & 0xE0) == 0x80) ||
|
|
|
80 (u == 0xF0 && (u2 & 0xF0) == 0x80) ||
|
|
|
81 (u == 0xF8 && (u2 & 0xF8) == 0x80) ||
|
|
|
82 (u == 0xFC && (u2 & 0xFC) == 0x80))
|
|
|
83 goto Lerr; // overlong combination
|
|
|
84
|
|
|
85 for (unsigned j = 1; j != n; j++)
|
|
|
86 {
|
|
|
87 u = s[i + j];
|
|
|
88 if ((u & 0xC0) != 0x80)
|
|
|
89 goto Lerr; // trailing bytes are 10xxxxxx
|
|
|
90 V = (V << 6) | (u & 0x3F);
|
|
|
91 }
|
|
|
92 if (!utf_isValidDchar(V))
|
|
|
93 goto Lerr;
|
|
|
94 i += n;
|
|
|
95 }
|
|
|
96 else
|
|
|
97 {
|
|
|
98 V = (dchar_t) u;
|
|
|
99 i++;
|
|
|
100 }
|
|
|
101
|
|
|
102 assert(utf_isValidDchar(V));
|
|
|
103 *pidx = i;
|
|
|
104 *presult = V;
|
|
|
105 return NULL;
|
|
|
106
|
|
|
107 Lerr:
|
|
|
108 *presult = (dchar_t) s[i];
|
|
|
109 *pidx = i + 1;
|
|
|
110 return "invalid UTF-8 sequence";
|
|
|
111 }
|
|
|
112
|
|
|
113 /***************************************************
|
|
|
114 * Validate a UTF-8 string.
|
|
|
115 * Returns:
|
|
|
116 * NULL success
|
|
|
117 * !=NULL error message string
|
|
|
118 */
|
|
|
119
|
|
|
120 char *utf_validateString(unsigned char *s, size_t len)
|
|
|
121 {
|
|
|
122 size_t idx;
|
|
|
123 char *err = NULL;
|
|
|
124 dchar_t dc;
|
|
|
125
|
|
|
126 for (idx = 0; idx < len; )
|
|
|
127 {
|
|
|
128 err = utf_decodeChar(s, len, &idx, &dc);
|
|
|
129 if (err)
|
|
|
130 break;
|
|
|
131 }
|
|
|
132 return err;
|
|
|
133 }
|
|
|
134
|
|
|
135
|
|
|
136 /********************************************
|
|
|
137 * Decode a single UTF-16 character sequence.
|
|
|
138 * Returns:
|
|
|
139 * NULL success
|
|
|
140 * !=NULL error message string
|
|
|
141 */
|
|
|
142
|
|
|
143
|
|
|
144 char *utf_decodeWchar(unsigned short *s, size_t len, size_t *pidx, dchar_t *presult)
|
|
|
145 {
|
|
|
146 char *msg;
|
|
|
147 size_t i = *pidx;
|
|
|
148 unsigned u = s[i];
|
|
|
149
|
|
|
150 assert(i >= 0 && i < len);
|
|
|
151 if (u & ~0x7F)
|
|
|
152 { if (u >= 0xD800 && u <= 0xDBFF)
|
|
|
153 { unsigned u2;
|
|
|
154
|
|
|
155 if (i + 1 == len)
|
|
|
156 { msg = "surrogate UTF-16 high value past end of string";
|
|
|
157 goto Lerr;
|
|
|
158 }
|
|
|
159 u2 = s[i + 1];
|
|
|
160 if (u2 < 0xDC00 || u2 > 0xDFFF)
|
|
|
161 { msg = "surrogate UTF-16 low value out of range";
|
|
|
162 goto Lerr;
|
|
|
163 }
|
|
|
164 u = ((u - 0xD7C0) << 10) + (u2 - 0xDC00);
|
|
|
165 i += 2;
|
|
|
166 }
|
|
|
167 else if (u >= 0xDC00 && u <= 0xDFFF)
|
|
|
168 { msg = "unpaired surrogate UTF-16 value";
|
|
|
169 goto Lerr;
|
|
|
170 }
|
|
|
171 else if (u == 0xFFFE || u == 0xFFFF)
|
|
|
172 { msg = "illegal UTF-16 value";
|
|
|
173 goto Lerr;
|
|
|
174 }
|
|
|
175 else
|
|
|
176 i++;
|
|
|
177 }
|
|
|
178 else
|
|
|
179 {
|
|
|
180 i++;
|
|
|
181 }
|
|
|
182
|
|
|
183 assert(utf_isValidDchar(u));
|
|
|
184 *pidx = i;
|
|
|
185 *presult = (dchar_t)u;
|
|
|
186 return NULL;
|
|
|
187
|
|
|
188 Lerr:
|
|
|
189 *presult = (dchar_t)s[i];
|
|
|
190 *pidx = i + 1;
|
|
|
191 return msg;
|
|
|
192 }
|
|
|
193
|
