0
|
1 module dmd.Token;
|
|
2
|
|
3 import dmd.TOK;
|
|
4 import dmd.Identifier;
|
|
5 import dmd.OutBuffer;
|
|
6 import dmd.Utf;
|
|
7
|
|
8 import core.stdc.stdio;
|
|
9 import core.stdc.ctype;
|
|
10
|
|
11 struct Token
|
|
12 {
|
|
13 Token* next;
|
|
14 ubyte* ptr; // pointer to first character of this token within buffer
|
|
15 TOK value;
|
|
16 ubyte* blockComment; // doc comment string prior to this token
|
|
17 ubyte* lineComment; // doc comment for previous token
|
|
18
|
|
19 union
|
|
20 {
|
|
21 // Integers
|
|
22 int int32value;
|
|
23 uint uns32value;
|
|
24 long int64value;
|
|
25 ulong uns64value;
|
|
26
|
|
27 // Floats
|
|
28 version (IN_GCC) {
|
|
29 // real_t float80value; // can't use this in a union!
|
|
30 } else {
|
|
31 real float80value;
|
|
32 }
|
|
33
|
|
34 struct
|
|
35 {
|
|
36 const(char)* ustring; // UTF8 string
|
|
37 uint len;
|
|
38 ubyte postfix; // 'c', 'w', 'd'
|
|
39 };
|
|
40
|
|
41 Identifier ident;
|
|
42 }
|
|
43
|
|
44 version (IN_GCC) {
|
|
45 real float80value; // can't use this in a union!
|
|
46 }
|
|
47
|
|
48 static string tochars[TOK.TOKMAX];
|
|
49 /// static void *operator new(size_t sz);
|
|
50
|
|
51 int isKeyword()
|
|
52 {
|
|
53 assert(false);
|
|
54 }
|
|
55
|
|
56 void print()
|
|
57 {
|
|
58 assert(false);
|
|
59 }
|
|
60
|
|
61 string toChars()
|
|
62 {
|
|
63 static char buffer[3 + 3 * value.sizeof + 1];
|
|
64
|
|
65 string p;
|
|
66
|
|
67 switch (value)
|
|
68 {
|
|
69 case TOK.TOKint32v:
|
|
70 version (IN_GCC) {
|
|
71 sprintf(buffer.ptr,"%d",cast(int)int64value);
|
|
72 } else {
|
|
73 sprintf(buffer.ptr,"%d",int32value);
|
|
74 }
|
|
75 break;
|
|
76
|
|
77 case TOK.TOKuns32v:
|
|
78 case TOK.TOKcharv:
|
|
79 case TOK.TOKwcharv:
|
|
80 case TOK.TOKdcharv:
|
|
81 version (IN_GCC) {
|
|
82 sprintf(buffer.ptr,"%uU",cast(uint)uns64value);
|
|
83 } else {
|
|
84 sprintf(buffer.ptr,"%uU",uns32value);
|
|
85 }
|
|
86 break;
|
|
87
|
|
88 case TOK.TOKint64v:
|
|
89 sprintf(buffer.ptr,"%jdL",int64value);
|
|
90 break;
|
|
91
|
|
92 case TOK.TOKuns64v:
|
|
93 sprintf(buffer.ptr,"%juUL",uns64value);
|
|
94 break;
|
|
95
|
|
96 version (IN_GCC) {
|
|
97 case TOK.TOKfloat32v:
|
|
98 case TOK.TOKfloat64v:
|
|
99 case TOK.TOKfloat80v:
|
|
100 float80value.format(buffer, sizeof(buffer));
|
|
101 break;
|
|
102 case TOK.TOKimaginary32v:
|
|
103 case TOK.TOKimaginary64v:
|
|
104 case TOK.TOKimaginary80v:
|
|
105 float80value.format(buffer, sizeof(buffer));
|
|
106 // %% buffer
|
|
107 strcat(buffer, "i");
|
|
108 break;
|
|
109 } else {
|
|
110 case TOK.TOKfloat32v:
|
|
111 sprintf(buffer.ptr,"%Lgf", float80value);
|
|
112 break;
|
|
113
|
|
114 case TOK.TOKfloat64v:
|
|
115 sprintf(buffer.ptr,"%Lg", float80value);
|
|
116 break;
|
|
117
|
|
118 case TOK.TOKfloat80v:
|
|
119 sprintf(buffer.ptr,"%LgL", float80value);
|
|
120 break;
|
|
121
|
|
122 case TOK.TOKimaginary32v:
|
|
123 sprintf(buffer.ptr,"%Lgfi", float80value);
|
|
124 break;
|
|
125
|
|
126 case TOK.TOKimaginary64v:
|
|
127 sprintf(buffer.ptr,"%Lgi", float80value);
|
|
128 break;
|
|
129
|
|
130 case TOK.TOKimaginary80v:
|
|
131 sprintf(buffer.ptr,"%LgLi", float80value);
|
|
132 break;
|
|
133 }
|
|
134
|
|
135 case TOK.TOKstring:
|
|
136 version (CSTRINGS) {
|
|
137 p = string;
|
|
138 } else {
|
|
139 { OutBuffer buf;
|
|
140
|
|
141 buf.writeByte('"');
|
|
142 for (size_t i = 0; i < len; )
|
|
143 {
|
|
144 dchar c; ///!
|
|
145
|
|
146 utf_decodeChar(ustring[0..len], &i, &c);
|
|
147 switch (c)
|
|
148 {
|
|
149 case 0:
|
|
150 break;
|
|
151
|
|
152 case '"':
|
|
153 case '\\':
|
|
154 buf.writeByte('\\');
|
|
155 default:
|
|
156 if (isprint(c))
|
|
157 buf.writeByte(c);
|
|
158 else if (c <= 0x7F)
|
|
159 buf.printf("\\x%02x", c);
|
|
160 else if (c <= 0xFFFF)
|
|
161 buf.printf("\\u%04x", c);
|
|
162 else
|
|
163 buf.printf("\\U%08x", c);
|
|
164 continue;
|
|
165 }
|
|
166 break;
|
|
167 }
|
|
168 buf.writeByte('"');
|
|
169 if (postfix)
|
|
170 buf.writeByte('"');
|
|
171 buf.writeByte(0);
|
|
172 p = buf.extractString();
|
|
173 }
|
|
174 }
|
|
175 break;
|
|
176
|
|
177 case TOK.TOKidentifier:
|
|
178 case TOK.TOKenum:
|
|
179 case TOK.TOKstruct:
|
|
180 case TOK.TOKimport:
|
|
181 case TOK.TOKwchar: case TOK.TOKdchar:
|
|
182 case TOK.TOKbit: case TOK.TOKbool: case TOK.TOKchar:
|
|
183 case TOK.TOKint8: case TOK.TOKuns8:
|
|
184 case TOK.TOKint16: case TOK.TOKuns16:
|
|
185 case TOK.TOKint32: case TOK.TOKuns32:
|
|
186 case TOK.TOKint64: case TOK.TOKuns64:
|
|
187 case TOK.TOKfloat32: case TOK.TOKfloat64: case TOK.TOKfloat80:
|
|
188 case TOK.TOKimaginary32: case TOK.TOKimaginary64: case TOK.TOKimaginary80:
|
|
189 case TOK.TOKcomplex32: case TOK.TOKcomplex64: case TOK.TOKcomplex80:
|
|
190 case TOK.TOKvoid:
|
|
191 p = ident.toChars();
|
|
192 break;
|
|
193
|
|
194 default:
|
|
195 p = toChars(value);
|
|
196 break;
|
|
197 }
|
|
198 return p;
|
|
199 }
|
|
200
|
|
201 static string toChars(TOK value)
|
|
202 {
|
|
203 string p;
|
|
204 static char buffer[3 + 3 * value.sizeof + 1];
|
|
205
|
|
206 p = tochars[value];
|
|
207 if (!p)
|
|
208 {
|
|
209 int len = sprintf(buffer.ptr, "TOK%d".ptr, value);
|
|
210 p = buffer[0..len].idup;
|
|
211 }
|
|
212
|
|
213 return p;
|
|
214 }
|
|
215 } |