Mercurial > projects > ldc

comparison dmd/utf.c @ 1:c53b6e3fe49a trunk

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
[svn r5] Initial commit. Most things are very rough.
author lindquist
date Sat, 01 Sep 2007 21:43:27 +0200
parents
children
comparison
equal deleted inserted replaced
0:a9e71648e74d 1:c53b6e3fe49a
1 // utf.c
2 // Copyright (c) 2003 by Digital Mars
3 // All Rights Reserved
4 // written by Walter Bright
5 // http://www.digitalmars.com
6 // License for redistribution is by either the Artistic License
7 // in artistic.txt, or the GNU General Public License in gnu.txt.
8 // See the included readme.txt for details.
9
10 // Description of UTF-8 at:
11 // http://www.cl.cam.ac.uk/~mgk25/unicode.html#utf-8
12
13 #include <stdio.h>
14 #include <assert.h>
15
16 #include "utf.h"
17
18 int utf_isValidDchar(dchar_t c)
19 {
20 return c < 0xD800 ||
21 (c > 0xDFFF && c <= 0x10FFFF && c != 0xFFFE && c != 0xFFFF);
22 }
23
24 /********************************************
25 * Decode a single UTF-8 character sequence.
26 * Returns:
27 * NULL success
28 * !=NULL error message string
29 */
30
31 char *utf_decodeChar(unsigned char *s, size_t len, size_t *pidx, dchar_t *presult)
32 {
33 dchar_t V;
34 size_t i = *pidx;
35 unsigned char u = s[i];
36
37 assert(i >= 0 && i < len);
38
39 if (u & 0x80)
40 { unsigned n;
41 unsigned char u2;
42
43 /* The following encodings are valid, except for the 5 and 6 byte
44 * combinations:
45 * 0xxxxxxx
46 * 110xxxxx 10xxxxxx
47 * 1110xxxx 10xxxxxx 10xxxxxx
48 * 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx
49 * 111110xx 10xxxxxx 10xxxxxx 10xxxxxx 10xxxxxx
50 * 1111110x 10xxxxxx 10xxxxxx 10xxxxxx 10xxxxxx 10xxxxxx
51 */
52 for (n = 1; ; n++)
53 {
54 if (n > 4)
55 goto Lerr; // only do the first 4 of 6 encodings
56 if (((u << n) & 0x80) == 0)
57 {
58 if (n == 1)
59 goto Lerr;
60 break;
61 }
62 }
63
64 // Pick off (7 - n) significant bits of B from first byte of octet
65 V = (dchar_t)(u & ((1 << (7 - n)) - 1));
66
67 if (i + (n - 1) >= len)
68 goto Lerr; // off end of string
69
70 /* The following combinations are overlong, and illegal:
71 * 1100000x (10xxxxxx)
72 * 11100000 100xxxxx (10xxxxxx)
73 * 11110000 1000xxxx (10xxxxxx 10xxxxxx)
74 * 11111000 10000xxx (10xxxxxx 10xxxxxx 10xxxxxx)
75 * 11111100 100000xx (10xxxxxx 10xxxxxx 10xxxxxx 10xxxxxx)
76 */
77 u2 = s[i + 1];
78 if ((u & 0xFE) == 0xC0 ||
79 (u == 0xE0 && (u2 & 0xE0) == 0x80) ||
80 (u == 0xF0 && (u2 & 0xF0) == 0x80) ||
81 (u == 0xF8 && (u2 & 0xF8) == 0x80) ||
82 (u == 0xFC && (u2 & 0xFC) == 0x80))
83 goto Lerr; // overlong combination
84
85 for (unsigned j = 1; j != n; j++)
86 {
87 u = s[i + j];
88 if ((u & 0xC0) != 0x80)
89 goto Lerr; // trailing bytes are 10xxxxxx
90 V = (V << 6) | (u & 0x3F);
91 }
92 if (!utf_isValidDchar(V))
93 goto Lerr;
94 i += n;
95 }
96 else
97 {
98 V = (dchar_t) u;
99 i++;
100 }
101
102 assert(utf_isValidDchar(V));
103 *pidx = i;
104 *presult = V;
105 return NULL;
106
107 Lerr:
108 *presult = (dchar_t) s[i];
109 *pidx = i + 1;
110 return "invalid UTF-8 sequence";
111 }
112
113 /***************************************************
114 * Validate a UTF-8 string.
115 * Returns:
116 * NULL success
117 * !=NULL error message string
118 */
119
120 char *utf_validateString(unsigned char *s, size_t len)
121 {
122 size_t idx;
123 char *err = NULL;
124 dchar_t dc;
125
126 for (idx = 0; idx < len; )
127 {
128 err = utf_decodeChar(s, len, &idx, &dc);
129 if (err)
130 break;
131 }
132 return err;
133 }
134
135
136 /********************************************
137 * Decode a single UTF-16 character sequence.
138 * Returns:
139 * NULL success
140 * !=NULL error message string
141 */
142
143
144 char *utf_decodeWchar(unsigned short *s, size_t len, size_t *pidx, dchar_t *presult)
145 {
146 char *msg;
147 size_t i = *pidx;
148 unsigned u = s[i];
149
150 assert(i >= 0 && i < len);
151 if (u & ~0x7F)
152 { if (u >= 0xD800 && u <= 0xDBFF)
153 { unsigned u2;
154
155 if (i + 1 == len)
156 { msg = "surrogate UTF-16 high value past end of string";
157 goto Lerr;
158 }
159 u2 = s[i + 1];
160 if (u2 < 0xDC00 || u2 > 0xDFFF)
161 { msg = "surrogate UTF-16 low value out of range";
162 goto Lerr;
163 }
164 u = ((u - 0xD7C0) << 10) + (u2 - 0xDC00);
165 i += 2;
166 }
167 else if (u >= 0xDC00 && u <= 0xDFFF)
168 { msg = "unpaired surrogate UTF-16 value";
169 goto Lerr;
170 }
171 else if (u == 0xFFFE || u == 0xFFFF)
172 { msg = "illegal UTF-16 value";
173 goto Lerr;
174 }
175 else
176 i++;
177 }
178 else
179 {
180 i++;
181 }
182
183 assert(utf_isValidDchar(u));
184 *pidx = i;
185 *presult = (dchar_t)u;
186 return NULL;
187
188 Lerr:
189 *presult = (dchar_t)s[i];
190 *pidx = i + 1;
191 return msg;
192 }
193