Mercurial > projects > dil
annotate trunk/src/dil/Unicode.d @ 741:35184354a502
Added method textBody() to IdentValueParser.
Applied some fixes.
author | Aziz K?ksal <aziz.koeksal@gmail.com> |
---|---|
date | Sat, 09 Feb 2008 16:18:53 +0100 |
parents | 49fe21aa387c |
children | 00f872d949ea |
rev | line source |
---|---|
510 | 1 /++ |
2 Author: Aziz Köksal | |
3 License: GPL3 | |
4 +/ | |
5 module dil.Unicode; | |
629
d050e211402b
Moved files in src/std/ to src/util/.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
518
diff
changeset
|
6 public import util.uni : isUniAlpha; |
510 | 7 |
8 /// U+FFFD = �. Used to replace invalid Unicode characters. | |
9 const dchar REPLACEMENT_CHAR = '\uFFFD'; | |
739
49fe21aa387c
Added sanitizeText() to dil.Converter.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
737
diff
changeset
|
10 const char[3] REPLACEMENT_STR = \uFFFD; /// Ditto |
510 | 11 /// Invalid character, returned on errors. |
12 const dchar ERROR_CHAR = 0xD800; | |
13 | |
14 /++ | |
15 Returns true if this character is not a surrogate | |
16 code point and not higher than 0x10FFFF. | |
17 +/ | |
18 bool isValidChar(dchar d) | |
19 { | |
20 return d < 0xD800 || d > 0xDFFF && d <= 0x10FFFF; | |
21 } | |
22 | |
23 /++ | |
24 There are a total of 66 noncharacters. | |
741
35184354a502
Added method textBody() to IdentValueParser.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
739
diff
changeset
|
25 Returns: true if this is one of them. |
510 | 26 See_also: Chapter 16.7 Noncharacters in Unicode 5.0 |
27 +/ | |
28 bool isNoncharacter(dchar d) | |
29 { | |
30 return 0xFDD0 <= d && d <= 0xFDEF || // 32 | |
31 d <= 0x10FFFF && (d & 0xFFFF) >= 0xFFFE; // 34 | |
32 } | |
33 | |
34 /// Returns true if this is a trail byte of a UTF-8 sequence? | |
35 bool isTrailByte(ubyte b) | |
36 { | |
37 return (b & 0xC0) == 0x80; // 10xx_xxxx | |
38 } | |
39 | |
40 /// Returns true if this is a lead byte of a UTF-8 sequence. | |
41 bool isLeadByte(ubyte b) | |
42 { | |
43 return (b & 0xC0) == 0xC0; // 11xx_xxxx | |
44 } | |
45 | |
722
ceaac6a24258
Added isUnicodeAlpha() for DDocParser and MacroParser.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
629
diff
changeset
|
46 /// Advances ref_p only if this is a valid Unicode alpha character. |
ceaac6a24258
Added isUnicodeAlpha() for DDocParser and MacroParser.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
629
diff
changeset
|
47 bool isUnicodeAlpha(ref char* ref_p, char* end) |
ceaac6a24258
Added isUnicodeAlpha() for DDocParser and MacroParser.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
629
diff
changeset
|
48 in { assert(ref_p && ref_p < end); } |
ceaac6a24258
Added isUnicodeAlpha() for DDocParser and MacroParser.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
629
diff
changeset
|
49 body |
ceaac6a24258
Added isUnicodeAlpha() for DDocParser and MacroParser.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
629
diff
changeset
|
50 { |
ceaac6a24258
Added isUnicodeAlpha() for DDocParser and MacroParser.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
629
diff
changeset
|
51 if (*ref_p < 0x80) |
ceaac6a24258
Added isUnicodeAlpha() for DDocParser and MacroParser.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
629
diff
changeset
|
52 return false; |
ceaac6a24258
Added isUnicodeAlpha() for DDocParser and MacroParser.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
629
diff
changeset
|
53 auto p = ref_p; |
ceaac6a24258
Added isUnicodeAlpha() for DDocParser and MacroParser.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
629
diff
changeset
|
54 auto c = decode(p, end); |
ceaac6a24258
Added isUnicodeAlpha() for DDocParser and MacroParser.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
629
diff
changeset
|
55 if (!isUniAlpha(c)) |
ceaac6a24258
Added isUnicodeAlpha() for DDocParser and MacroParser.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
629
diff
changeset
|
56 return false; |
ceaac6a24258
Added isUnicodeAlpha() for DDocParser and MacroParser.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
629
diff
changeset
|
57 ref_p = p; |
ceaac6a24258
Added isUnicodeAlpha() for DDocParser and MacroParser.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
629
diff
changeset
|
58 return true; |
ceaac6a24258
Added isUnicodeAlpha() for DDocParser and MacroParser.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
629
diff
changeset
|
59 } |
ceaac6a24258
Added isUnicodeAlpha() for DDocParser and MacroParser.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
629
diff
changeset
|
60 |
ceaac6a24258
Added isUnicodeAlpha() for DDocParser and MacroParser.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
629
diff
changeset
|
61 /// index is set one past the last trail byte of the valid UTF-8 sequence. |
510 | 62 dchar decode(char[] str, ref size_t index) |
722
ceaac6a24258
Added isUnicodeAlpha() for DDocParser and MacroParser.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
629
diff
changeset
|
63 in { assert(str.length && index < str.length); } |
737
f88b5285b86b
Implemented DDocEmitter.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
722
diff
changeset
|
64 out(c) { assert(isValidChar(c) || c == ERROR_CHAR); } |
510 | 65 body |
66 { | |
67 char* p = str.ptr + index; | |
68 char* end = str.ptr + str.length; | |
722
ceaac6a24258
Added isUnicodeAlpha() for DDocParser and MacroParser.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
629
diff
changeset
|
69 dchar c = decode(p, end); |
ceaac6a24258
Added isUnicodeAlpha() for DDocParser and MacroParser.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
629
diff
changeset
|
70 if (c != ERROR_CHAR) |
ceaac6a24258
Added isUnicodeAlpha() for DDocParser and MacroParser.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
629
diff
changeset
|
71 index = p - str.ptr + 1; |
ceaac6a24258
Added isUnicodeAlpha() for DDocParser and MacroParser.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
629
diff
changeset
|
72 return c; |
ceaac6a24258
Added isUnicodeAlpha() for DDocParser and MacroParser.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
629
diff
changeset
|
73 } |
510 | 74 |
722
ceaac6a24258
Added isUnicodeAlpha() for DDocParser and MacroParser.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
629
diff
changeset
|
75 /// ref_p is set to the last trail byte of the valid UTF-8 sequence. |
ceaac6a24258
Added isUnicodeAlpha() for DDocParser and MacroParser.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
629
diff
changeset
|
76 dchar decode(ref char* ref_p, char* end) |
ceaac6a24258
Added isUnicodeAlpha() for DDocParser and MacroParser.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
629
diff
changeset
|
77 in { assert(ref_p && ref_p < end); } |
737
f88b5285b86b
Implemented DDocEmitter.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
722
diff
changeset
|
78 out(c) { assert(isValidChar(c) || c == ERROR_CHAR); } |
722
ceaac6a24258
Added isUnicodeAlpha() for DDocParser and MacroParser.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
629
diff
changeset
|
79 body |
ceaac6a24258
Added isUnicodeAlpha() for DDocParser and MacroParser.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
629
diff
changeset
|
80 { |
ceaac6a24258
Added isUnicodeAlpha() for DDocParser and MacroParser.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
629
diff
changeset
|
81 char* p = ref_p; |
ceaac6a24258
Added isUnicodeAlpha() for DDocParser and MacroParser.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
629
diff
changeset
|
82 dchar c = *p; |
510 | 83 |
84 if (c < 0x80) | |
85 { | |
722
ceaac6a24258
Added isUnicodeAlpha() for DDocParser and MacroParser.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
629
diff
changeset
|
86 ref_p++; |
510 | 87 return c; |
88 } | |
89 | |
722
ceaac6a24258
Added isUnicodeAlpha() for DDocParser and MacroParser.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
629
diff
changeset
|
90 p++; // Move to second byte. |
510 | 91 if (!(p < end)) |
92 return ERROR_CHAR; | |
93 | |
94 // Error if second byte is not a trail byte. | |
95 if (!isTrailByte(*p)) | |
96 return ERROR_CHAR; | |
97 | |
98 // Check for overlong sequences. | |
99 switch (c) | |
100 { | |
101 case 0xE0, // 11100000 100xxxxx | |
102 0xF0, // 11110000 1000xxxx | |
103 0xF8, // 11111000 10000xxx | |
104 0xFC: // 11111100 100000xx | |
105 if ((*p & c) == 0x80) | |
106 return ERROR_CHAR; | |
107 default: | |
108 if ((c & 0xFE) == 0xC0) // 1100000x | |
109 return ERROR_CHAR; | |
110 } | |
111 | |
722
ceaac6a24258
Added isUnicodeAlpha() for DDocParser and MacroParser.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
629
diff
changeset
|
112 const char[] checkNextByte = "if (!(++p < end && isTrailByte(*p)))" |
510 | 113 " return ERROR_CHAR;"; |
114 const char[] appendSixBits = "c = (c << 6) | *p & 0b0011_1111;"; | |
115 | |
116 // Decode | |
117 if ((c & 0b1110_0000) == 0b1100_0000) | |
118 { | |
119 // 110xxxxx 10xxxxxx | |
120 c &= 0b0001_1111; | |
121 mixin(appendSixBits); | |
122 } | |
123 else if ((c & 0b1111_0000) == 0b1110_0000) | |
124 { | |
125 // 1110xxxx 10xxxxxx 10xxxxxx | |
126 c &= 0b0000_1111; | |
127 mixin(appendSixBits ~ | |
128 checkNextByte ~ appendSixBits); | |
129 } | |
130 else if ((c & 0b1111_1000) == 0b1111_0000) | |
131 { | |
132 // 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx | |
133 c &= 0b0000_0111; | |
134 mixin(appendSixBits ~ | |
135 checkNextByte ~ appendSixBits ~ | |
136 checkNextByte ~ appendSixBits); | |
137 } | |
138 else | |
139 // 5 and 6 byte UTF-8 sequences are not allowed yet. | |
140 // 111110xx 10xxxxxx 10xxxxxx 10xxxxxx 10xxxxxx | |
141 // 1111110x 10xxxxxx 10xxxxxx 10xxxxxx 10xxxxxx 10xxxxxx | |
142 return ERROR_CHAR; | |
143 | |
144 assert(isTrailByte(*p)); | |
145 | |
146 if (!isValidChar(c)) | |
147 return ERROR_CHAR; | |
722
ceaac6a24258
Added isUnicodeAlpha() for DDocParser and MacroParser.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
629
diff
changeset
|
148 ref_p = p; |
510 | 149 return c; |
150 } | |
151 | |
152 /// Encodes a character and appends it to str. | |
518
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
510
diff
changeset
|
153 void encode(ref char[] str, dchar c) |
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
510
diff
changeset
|
154 { |
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
510
diff
changeset
|
155 assert(isValidChar(c), "check if character is valid before calling encode()."); |
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
510
diff
changeset
|
156 |
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
510
diff
changeset
|
157 char[6] b = void; |
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
510
diff
changeset
|
158 if (c < 0x80) |
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
510
diff
changeset
|
159 str ~= c; |
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
510
diff
changeset
|
160 if (c < 0x800) |
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
510
diff
changeset
|
161 { |
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
510
diff
changeset
|
162 b[0] = 0xC0 | (c >> 6); |
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
510
diff
changeset
|
163 b[1] = 0x80 | (c & 0x3F); |
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
510
diff
changeset
|
164 str ~= b[0..2]; |
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
510
diff
changeset
|
165 } |
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
510
diff
changeset
|
166 else if (c < 0x10000) |
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
510
diff
changeset
|
167 { |
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
510
diff
changeset
|
168 b[0] = 0xE0 | (c >> 12); |
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
510
diff
changeset
|
169 b[1] = 0x80 | ((c >> 6) & 0x3F); |
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
510
diff
changeset
|
170 b[2] = 0x80 | (c & 0x3F); |
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
510
diff
changeset
|
171 str ~= b[0..3]; |
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
510
diff
changeset
|
172 } |
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
510
diff
changeset
|
173 else if (c < 0x200000) |
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
510
diff
changeset
|
174 { |
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
510
diff
changeset
|
175 b[0] = 0xF0 | (c >> 18); |
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
510
diff
changeset
|
176 b[1] = 0x80 | ((c >> 12) & 0x3F); |
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
510
diff
changeset
|
177 b[2] = 0x80 | ((c >> 6) & 0x3F); |
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
510
diff
changeset
|
178 b[3] = 0x80 | (c & 0x3F); |
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
510
diff
changeset
|
179 str ~= b[0..4]; |
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
510
diff
changeset
|
180 } |
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
510
diff
changeset
|
181 /+ // There are no 5 and 6 byte UTF-8 sequences yet. |
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
510
diff
changeset
|
182 else if (c < 0x4000000) |
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
510
diff
changeset
|
183 { |
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
510
diff
changeset
|
184 b[0] = 0xF8 | (c >> 24); |
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
510
diff
changeset
|
185 b[1] = 0x80 | ((c >> 18) & 0x3F); |
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
510
diff
changeset
|
186 b[2] = 0x80 | ((c >> 12) & 0x3F); |
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
510
diff
changeset
|
187 b[3] = 0x80 | ((c >> 6) & 0x3F); |
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
510
diff
changeset
|
188 b[4] = 0x80 | (c & 0x3F); |
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
510
diff
changeset
|
189 str ~= b[0..5]; |
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
510
diff
changeset
|
190 } |
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
510
diff
changeset
|
191 else if (c < 0x80000000) |
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
510
diff
changeset
|
192 { |
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
510
diff
changeset
|
193 b[0] = 0xFC | (c >> 30); |
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
510
diff
changeset
|
194 b[1] = 0x80 | ((c >> 24) & 0x3F); |
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
510
diff
changeset
|
195 b[2] = 0x80 | ((c >> 18) & 0x3F); |
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
510
diff
changeset
|
196 b[3] = 0x80 | ((c >> 12) & 0x3F); |
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
510
diff
changeset
|
197 b[4] = 0x80 | ((c >> 6) & 0x3F); |
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
510
diff
changeset
|
198 b[5] = 0x80 | (c & 0x3F); |
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
510
diff
changeset
|
199 str ~= b[0..6]; |
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
510
diff
changeset
|
200 } |
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
510
diff
changeset
|
201 +/ |
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
510
diff
changeset
|
202 else |
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
510
diff
changeset
|
203 assert(0); |
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
510
diff
changeset
|
204 } |
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
510
diff
changeset
|
205 |
8f86bb9ef715
Added module dil.Converter and dil.FileBOM.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
510
diff
changeset
|
206 /// Encodes a character and appends it to str. |
510 | 207 void encode(ref wchar[] str, dchar c) |
208 in { assert(isValidChar(c)); } | |
209 body | |
210 { | |
211 if (c < 0x10000) | |
212 str ~= cast(wchar)c; | |
213 else | |
214 { | |
215 // Encode with surrogate pair. | |
216 wchar[2] pair = void; | |
217 c -= 0x10000; // c' | |
218 // higher10bits(c') | 0b1101_10xx_xxxx_xxxx | |
219 pair[0] = (c >> 10) | 0xD800; | |
220 // lower10bits(c') | 0b1101_11yy_yyyy_yyyy | |
221 pair[1] = (c & 0x3FF) | 0xDC00; | |
222 str ~= pair; | |
223 } | |
224 } | |
225 | |
226 /++ | |
227 Returns a decoded character from a UTF-16 sequence. | |
228 In case of an error in the sequence 0xD800 is returned. | |
229 Params: | |
230 str = the UTF-16 sequence. | |
231 index = where to start from. | |
232 +/ | |
233 dchar decode(wchar[] str, ref size_t index) | |
234 { | |
235 assert(str.length && index < str.length); | |
236 dchar c = str[index]; | |
237 if (0xD800 > c || c > 0xDFFF) | |
238 { | |
239 ++index; | |
240 return c; | |
241 } | |
242 if (c <= 0xDBFF && index+1 != str.length) | |
243 { | |
244 wchar c2 = str[index+1]; | |
245 if (0xDC00 <= c2 && c2 <= 0xDFFF) | |
246 { | |
247 // (c - 0xD800) << 10 + 0x10000 -> | |
248 // (c - 0xD800 + 0x40) << 10 -> | |
249 c = (c - 0xD7C0) << 10; | |
250 c |= (c2 & 0x3FF); | |
251 index += 2; | |
252 return c; | |
253 } | |
254 } | |
255 return ERROR_CHAR; | |
256 } | |
257 | |
258 /++ | |
259 Returns a decoded character from a UTF-16 sequence. | |
260 In case of an error in the sequence 0xD800 is returned. | |
261 Params: | |
262 p = start of the UTF-16 sequence. | |
263 end = one past the end of the sequence. | |
264 +/ | |
265 dchar decode(ref wchar* p, wchar* end) | |
266 { | |
267 assert(p && p < end); | |
268 dchar c = *p; | |
269 if (0xD800 > c || c > 0xDFFF) | |
270 { | |
271 ++p; | |
272 return c; | |
273 } | |
274 if (c <= 0xDBFF && p+1 != end) | |
275 { | |
276 wchar c2 = p[1]; | |
277 if (0xDC00 <= c2 && c2 <= 0xDFFF) | |
278 { | |
279 c = (c - 0xD7C0) << 10; | |
280 c |= (c2 & 0x3FF); | |
281 p += 2; | |
282 return c; | |
283 } | |
284 } | |
285 return ERROR_CHAR; | |
286 } | |
287 | |
288 /// Decode a character from a zero-terminated string. | |
289 dchar decode(ref wchar* p) | |
290 { | |
291 assert(p); | |
292 dchar c = *p; | |
293 if (0xD800 > c || c > 0xDFFF) | |
294 { | |
295 ++p; | |
296 return c; | |
297 } | |
298 if (c <= 0xDBFF) | |
299 { | |
300 wchar c2 = p[1]; | |
301 if (0xDC00 <= c2 && c2 <= 0xDFFF) | |
302 { | |
303 c = (c - 0xD7C0) << 10; | |
304 c |= (c2 & 0x3FF); | |
305 p += 2; | |
306 return c; | |
307 } | |
308 } | |
309 return ERROR_CHAR; | |
310 } |