Mercurial > projects > ddmd

comparison dmd/Token.d @ 0:10317f0c89a5

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Initial commit
author korDen
date Sat, 24 Oct 2009 08:42:06 +0400
parents
children 5c9b78899f5d
comparison
equal deleted inserted replaced
-1:000000000000 0:10317f0c89a5
1 module dmd.Token;
2
3 import dmd.TOK;
4 import dmd.Identifier;
5 import dmd.OutBuffer;
6 import dmd.Utf;
7
8 import core.stdc.stdio;
9 import core.stdc.ctype;
10
11 struct Token
12 {
13 Token* next;
14 ubyte* ptr; // pointer to first character of this token within buffer
15 TOK value;
16 ubyte* blockComment; // doc comment string prior to this token
17 ubyte* lineComment; // doc comment for previous token
18
19 union
20 {
21 // Integers
22 int int32value;
23 uint uns32value;
24 long int64value;
25 ulong uns64value;
26
27 // Floats
28 version (IN_GCC) {
29 // real_t float80value; // can't use this in a union!
30 } else {
31 real float80value;
32 }
33
34 struct
35 {
36 const(char)* ustring; // UTF8 string
37 uint len;
38 ubyte postfix; // 'c', 'w', 'd'
39 };
40
41 Identifier ident;
42 }
43
44 version (IN_GCC) {
45 real float80value; // can't use this in a union!
46 }
47
48 static string tochars[TOK.TOKMAX];
49 /// static void *operator new(size_t sz);
50
51 int isKeyword()
52 {
53 assert(false);
54 }
55
56 void print()
57 {
58 assert(false);
59 }
60
61 string toChars()
62 {
63 static char buffer[3 + 3 * value.sizeof + 1];
64
65 string p;
66
67 switch (value)
68 {
69 case TOK.TOKint32v:
70 version (IN_GCC) {
71 sprintf(buffer.ptr,"%d",cast(int)int64value);
72 } else {
73 sprintf(buffer.ptr,"%d",int32value);
74 }
75 break;
76
77 case TOK.TOKuns32v:
78 case TOK.TOKcharv:
79 case TOK.TOKwcharv:
80 case TOK.TOKdcharv:
81 version (IN_GCC) {
82 sprintf(buffer.ptr,"%uU",cast(uint)uns64value);
83 } else {
84 sprintf(buffer.ptr,"%uU",uns32value);
85 }
86 break;
87
88 case TOK.TOKint64v:
89 sprintf(buffer.ptr,"%jdL",int64value);
90 break;
91
92 case TOK.TOKuns64v:
93 sprintf(buffer.ptr,"%juUL",uns64value);
94 break;
95
96 version (IN_GCC) {
97 case TOK.TOKfloat32v:
98 case TOK.TOKfloat64v:
99 case TOK.TOKfloat80v:
100 float80value.format(buffer, sizeof(buffer));
101 break;
102 case TOK.TOKimaginary32v:
103 case TOK.TOKimaginary64v:
104 case TOK.TOKimaginary80v:
105 float80value.format(buffer, sizeof(buffer));
106 // %% buffer
107 strcat(buffer, "i");
108 break;
109 } else {
110 case TOK.TOKfloat32v:
111 sprintf(buffer.ptr,"%Lgf", float80value);
112 break;
113
114 case TOK.TOKfloat64v:
115 sprintf(buffer.ptr,"%Lg", float80value);
116 break;
117
118 case TOK.TOKfloat80v:
119 sprintf(buffer.ptr,"%LgL", float80value);
120 break;
121
122 case TOK.TOKimaginary32v:
123 sprintf(buffer.ptr,"%Lgfi", float80value);
124 break;
125
126 case TOK.TOKimaginary64v:
127 sprintf(buffer.ptr,"%Lgi", float80value);
128 break;
129
130 case TOK.TOKimaginary80v:
131 sprintf(buffer.ptr,"%LgLi", float80value);
132 break;
133 }
134
135 case TOK.TOKstring:
136 version (CSTRINGS) {
137 p = string;
138 } else {
139 { OutBuffer buf;
140
141 buf.writeByte('"');
142 for (size_t i = 0; i < len; )
143 {
144 dchar c; ///!
145
146 utf_decodeChar(ustring[0..len], &i, &c);
147 switch (c)
148 {
149 case 0:
150 break;
151
152 case '"':
153 case '\\':
154 buf.writeByte('\\');
155 default:
156 if (isprint(c))
157 buf.writeByte(c);
158 else if (c <= 0x7F)
159 buf.printf("\\x%02x", c);
160 else if (c <= 0xFFFF)
161 buf.printf("\\u%04x", c);
162 else
163 buf.printf("\\U%08x", c);
164 continue;
165 }
166 break;
167 }
168 buf.writeByte('"');
169 if (postfix)
170 buf.writeByte('"');
171 buf.writeByte(0);
172 p = buf.extractString();
173 }
174 }
175 break;
176
177 case TOK.TOKidentifier:
178 case TOK.TOKenum:
179 case TOK.TOKstruct:
180 case TOK.TOKimport:
181 case TOK.TOKwchar: case TOK.TOKdchar:
182 case TOK.TOKbit: case TOK.TOKbool: case TOK.TOKchar:
183 case TOK.TOKint8: case TOK.TOKuns8:
184 case TOK.TOKint16: case TOK.TOKuns16:
185 case TOK.TOKint32: case TOK.TOKuns32:
186 case TOK.TOKint64: case TOK.TOKuns64:
187 case TOK.TOKfloat32: case TOK.TOKfloat64: case TOK.TOKfloat80:
188 case TOK.TOKimaginary32: case TOK.TOKimaginary64: case TOK.TOKimaginary80:
189 case TOK.TOKcomplex32: case TOK.TOKcomplex64: case TOK.TOKcomplex80:
190 case TOK.TOKvoid:
191 p = ident.toChars();
192 break;
193
194 default:
195 p = toChars(value);
196 break;
197 }
198 return p;
199 }
200
201 static string toChars(TOK value)
202 {
203 string p;
204 static char buffer[3 + 3 * value.sizeof + 1];
205
206 p = tochars[value];
207 if (!p)
208 {
209 int len = sprintf(buffer.ptr, "TOK%d".ptr, value);
210 p = buffer[0..len].idup;
211 }
212
213 return p;
214 }
215 }