Mercurial > projects > dil

annotate trunk/src/Token.d @ 312:fa0b6f32c1ae

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
- Added Special to enum TOK. - Added special tokens to keyword list. - Added case T.Special for parsing SpecialTokenExpression.
author aziz
date Wed, 15 Aug 2007 19:19:00 +0000
parents a416e09c08ea
children ebd21bbf296e
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
0
8ba2570de175 Initial import.
aziz
parents:
diff changeset
1 /++
8ba2570de175 Initial import.
aziz
parents:
diff changeset
2 Author: Aziz Köksal
249
32d354584b28 - Upgraded license notices to GPL3.
aziz
parents: 239
diff changeset
3 License: GPL3
0
8ba2570de175 Initial import.
aziz
parents:
diff changeset
4 +/
8ba2570de175 Initial import.
aziz
parents:
diff changeset
5 module Token;
239
7911f6a92e6e - Added 'new' and 'delete' declarations to Token and uncommented next and prev members. Added HEAD to TOK.
aziz
parents: 210
diff changeset
6 import std.c.stdlib : malloc, free;
7911f6a92e6e - Added 'new' and 'delete' declarations to Token and uncommented next and prev members. Added HEAD to TOK.
aziz
parents: 210
diff changeset
7 import std.outofmemory;
0
8ba2570de175 Initial import.
aziz
parents:
diff changeset
8
8ba2570de175 Initial import.
aziz
parents:
diff changeset
9 struct Position
8ba2570de175 Initial import.
aziz
parents:
diff changeset
10 {
8ba2570de175 Initial import.
aziz
parents:
diff changeset
11 size_t loc;
8ba2570de175 Initial import.
aziz
parents:
diff changeset
12 size_t col;
8ba2570de175 Initial import.
aziz
parents:
diff changeset
13 }
8ba2570de175 Initial import.
aziz
parents:
diff changeset
14
3
4bbce78bfb1e - Added TOK enum.
aziz
parents: 0
diff changeset
15 enum TOK
4bbce78bfb1e - Added TOK enum.
aziz
parents: 0
diff changeset
16 {
95
0eb4c8a5b32b - Added TOK.Invalid.
aziz
parents: 84
diff changeset
17 Invalid,
0eb4c8a5b32b - Added TOK.Invalid.
aziz
parents: 84
diff changeset
18
3
4bbce78bfb1e - Added TOK enum.
aziz
parents: 0
diff changeset
19 Identifier,
4
92df59b1ec4a - Started implementation of scan().
aziz
parents: 3
diff changeset
20 Comment,
9
5d6968cc751e - Parsing string and character literals now (rudimentary implementation.)
aziz
parents: 4
diff changeset
21 String,
312
fa0b6f32c1ae - Added Special to enum TOK.
aziz
parents: 269
diff changeset
22 Special,
82
fc645fb2fe72 - scanEscapeSequences() doesn't return 0xFFFF as an error value anymore, because it is a valid codepoint usable by the user.
aziz
parents: 71
diff changeset
23 CharLiteral, WCharLiteral, DCharLiteral,
58
50bb7fc9db44 - The types of integers are recognized now.
aziz
parents: 56
diff changeset
24
50bb7fc9db44 - The types of integers are recognized now.
aziz
parents: 56
diff changeset
25 // Numbers
50bb7fc9db44 - The types of integers are recognized now.
aziz
parents: 56
diff changeset
26 Int32, Int64, Uint32, Uint64,
97
1a83e5070a84 - Added code for parsing IntNumber- and RealNumberExpressions.
aziz
parents: 95
diff changeset
27 // Floating point number scanner relies on this order. (FloatXY + 3 == ImaginaryXY)
62
96af5653acef - Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents: 58
diff changeset
28 Float32, Float64, Float80,
96af5653acef - Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents: 58
diff changeset
29 Imaginary32, Imaginary64, Imaginary80,
58
50bb7fc9db44 - The types of integers are recognized now.
aziz
parents: 56
diff changeset
30
36
3c7210a722f7 - Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents: 35
diff changeset
31
3c7210a722f7 - Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents: 35
diff changeset
32 // Brackets
20
d6adfbd7c513 - Added code for parsing braces.
aziz
parents: 18
diff changeset
33 LParen,
d6adfbd7c513 - Added code for parsing braces.
aziz
parents: 18
diff changeset
34 RParen,
d6adfbd7c513 - Added code for parsing braces.
aziz
parents: 18
diff changeset
35 LBracket,
d6adfbd7c513 - Added code for parsing braces.
aziz
parents: 18
diff changeset
36 RBracket,
d6adfbd7c513 - Added code for parsing braces.
aziz
parents: 18
diff changeset
37 LBrace,
d6adfbd7c513 - Added code for parsing braces.
aziz
parents: 18
diff changeset
38 RBrace,
21
c785c122e4e6 - Added code for parsing Colon, Semicolon, Question, Comma and Dollar.
aziz
parents: 20
diff changeset
39
22
b05fff8e2ce4 - Added code for parsing Dot, Slice and Ellipses tokens.
aziz
parents: 21
diff changeset
40 Dot, Slice, Ellipses,
b05fff8e2ce4 - Added code for parsing Dot, Slice and Ellipses tokens.
aziz
parents: 21
diff changeset
41
131
ce636f3981cc - Removed TOK.Number.
aziz
parents: 107
diff changeset
42 // Floating point number operators
35
c470b9356e35 - Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents: 32
diff changeset
43 Unordered,
c470b9356e35 - Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents: 32
diff changeset
44 UorE,
c470b9356e35 - Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents: 32
diff changeset
45 UorG,
c470b9356e35 - Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents: 32
diff changeset
46 UorGorE,
c470b9356e35 - Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents: 32
diff changeset
47 UorL,
c470b9356e35 - Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents: 32
diff changeset
48 UorLorE,
36
3c7210a722f7 - Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents: 35
diff changeset
49 LorEorG,
3c7210a722f7 - Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents: 35
diff changeset
50 LorG,
35
c470b9356e35 - Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents: 32
diff changeset
51
36
3c7210a722f7 - Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents: 35
diff changeset
52 // Normal operators
35
c470b9356e35 - Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents: 32
diff changeset
53 Assign, Equal, NotEqual, Not,
38
640c45aaaaee - Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents: 36
diff changeset
54 LessEqual, Less,
640c45aaaaee - Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents: 36
diff changeset
55 GreaterEqual, Greater,
36
3c7210a722f7 - Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents: 35
diff changeset
56 LShiftAssign, LShift,
38
640c45aaaaee - Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents: 36
diff changeset
57 RShiftAssign,RShift,
640c45aaaaee - Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents: 36
diff changeset
58 URShiftAssign, URShift,
23
1a7903701a3d - Added code for parsing OrAssign, OrLogical and OrBinary tokens.
aziz
parents: 22
diff changeset
59 OrAssign, OrLogical, OrBinary,
24
903f91163f23 - Added code for parsing AndAssign, AndLogical and AndBinary tokens.
aziz
parents: 23
diff changeset
60 AndAssign, AndLogical, AndBinary,
25
9c866208b3f6 - Added code for parsing PlusAssign, PlusPlus, Plus, MinusAssign, MinusMinus, Minus tokens.
aziz
parents: 24
diff changeset
61 PlusAssign, PlusPlus, Plus,
9c866208b3f6 - Added code for parsing PlusAssign, PlusPlus, Plus, MinusAssign, MinusMinus, Minus tokens.
aziz
parents: 24
diff changeset
62 MinusAssign, MinusMinus, Minus,
32
d7011daa4740 - Added missing commas after the items in the messages table.
aziz
parents: 31
diff changeset
63 DivAssign, Div,
29
ef83eea26bbd - Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents: 28
diff changeset
64 MulAssign, Mul,
ef83eea26bbd - Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents: 28
diff changeset
65 ModAssign, Mod,
ef83eea26bbd - Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents: 28
diff changeset
66 XorAssign, Xor,
27
43b6bf56f0e9 - Added code for parsing CatAssign and Tilde tokens.
aziz
parents: 26
diff changeset
67 CatAssign, Catenate,
39
69b940398d7b - Added unittest to test correct parsing of operator tokens.
aziz
parents: 38
diff changeset
68 Tilde,
71
b3777cca323c - Added Identity and NotIdentity tokens.
aziz
parents: 65
diff changeset
69 Identity, NotIdentity,
23
1a7903701a3d - Added code for parsing OrAssign, OrLogical and OrBinary tokens.
aziz
parents: 22
diff changeset
70
21
c785c122e4e6 - Added code for parsing Colon, Semicolon, Question, Comma and Dollar.
aziz
parents: 20
diff changeset
71 Colon,
c785c122e4e6 - Added code for parsing Colon, Semicolon, Question, Comma and Dollar.
aziz
parents: 20
diff changeset
72 Semicolon,
c785c122e4e6 - Added code for parsing Colon, Semicolon, Question, Comma and Dollar.
aziz
parents: 20
diff changeset
73 Question,
c785c122e4e6 - Added code for parsing Colon, Semicolon, Question, Comma and Dollar.
aziz
parents: 20
diff changeset
74 Comma,
c785c122e4e6 - Added code for parsing Colon, Semicolon, Question, Comma and Dollar.
aziz
parents: 20
diff changeset
75 Dollar,
c785c122e4e6 - Added code for parsing Colon, Semicolon, Question, Comma and Dollar.
aziz
parents: 20
diff changeset
76
28
3a9daccf7d96 - Added table for identifiers to Lexer.
aziz
parents: 27
diff changeset
77 /* Keywords:
3a9daccf7d96 - Added table for identifiers to Lexer.
aziz
parents: 27
diff changeset
78 NB.: Token.isKeyword() depends on this list being contiguous.
3a9daccf7d96 - Added table for identifiers to Lexer.
aziz
parents: 27
diff changeset
79 */
3a9daccf7d96 - Added table for identifiers to Lexer.
aziz
parents: 27
diff changeset
80 Abstract,Alias,Align,Asm,Assert,Auto,Body,
3a9daccf7d96 - Added table for identifiers to Lexer.
aziz
parents: 27
diff changeset
81 Bool,Break,Byte,Case,Cast,Catch,Cdouble,
3a9daccf7d96 - Added table for identifiers to Lexer.
aziz
parents: 27
diff changeset
82 Cent,Cfloat,Char,Class,Const,Continue,Creal,
3a9daccf7d96 - Added table for identifiers to Lexer.
aziz
parents: 27
diff changeset
83 Dchar,Debug,Default,Delegate,Delete,Deprecated,Do,
3a9daccf7d96 - Added table for identifiers to Lexer.
aziz
parents: 27
diff changeset
84 Double,Else,Enum,Export,Extern,False,Final,
3a9daccf7d96 - Added table for identifiers to Lexer.
aziz
parents: 27
diff changeset
85 Finally,Float,For,Foreach,Foreach_reverse,Function,Goto,
3a9daccf7d96 - Added table for identifiers to Lexer.
aziz
parents: 27
diff changeset
86 Idouble,If,Ifloat,Import,In,Inout,Int,
269
a416e09c08ea - Implemented D 2.0 additions.
aziz
parents: 249
diff changeset
87 Interface,Invariant,Ireal,Is,Lazy,Long,Macro/+D2.0+/,
28
3a9daccf7d96 - Added table for identifiers to Lexer.
aziz
parents: 27
diff changeset
88 Mixin,Module,New,Null,Out,Override,Package,
269
a416e09c08ea - Implemented D 2.0 additions.
aziz
parents: 249
diff changeset
89 Pragma,Private,Protected,Public,Real,Ref/+D2.0+/,Return,
28
3a9daccf7d96 - Added table for identifiers to Lexer.
aziz
parents: 27
diff changeset
90 Scope,Short,Static,Struct,Super,Switch,Synchronized,
269
a416e09c08ea - Implemented D 2.0 additions.
aziz
parents: 249
diff changeset
91 Template,This,Throw,Traits/+D2.0+/,True,Try,Typedef,Typeid,
28
3a9daccf7d96 - Added table for identifiers to Lexer.
aziz
parents: 27
diff changeset
92 Typeof,Ubyte,Ucent,Uint,Ulong,Union,Unittest,
3a9daccf7d96 - Added table for identifiers to Lexer.
aziz
parents: 27
diff changeset
93 Ushort,Version,Void,Volatile,Wchar,While,With,
3a9daccf7d96 - Added table for identifiers to Lexer.
aziz
parents: 27
diff changeset
94
239
7911f6a92e6e - Added 'new' and 'delete' declarations to Token and uncommented next and prev members. Added HEAD to TOK.
aziz
parents: 210
diff changeset
95 HEAD, // start of linked list
4
92df59b1ec4a - Started implementation of scan().
aziz
parents: 3
diff changeset
96 EOF
3
4bbce78bfb1e - Added TOK enum.
aziz
parents: 0
diff changeset
97 }
4bbce78bfb1e - Added TOK enum.
aziz
parents: 0
diff changeset
98
28
3a9daccf7d96 - Added table for identifiers to Lexer.
aziz
parents: 27
diff changeset
99 alias TOK.Abstract KeywordsBegin;
3a9daccf7d96 - Added table for identifiers to Lexer.
aziz
parents: 27
diff changeset
100 alias TOK.With KeywordsEnd;
3a9daccf7d96 - Added table for identifiers to Lexer.
aziz
parents: 27
diff changeset
101
0
8ba2570de175 Initial import.
aziz
parents:
diff changeset
102 struct Token
8ba2570de175 Initial import.
aziz
parents:
diff changeset
103 {
3
4bbce78bfb1e - Added TOK enum.
aziz
parents: 0
diff changeset
104 TOK type;
84
ac8d961d10d1 - Added code for parsing This-,Super-,Null-,Bool-,Dollar-,CharLiteral- and StringLiteralExpression.
aziz
parents: 82
diff changeset
105 // Position pos;
0
8ba2570de175 Initial import.
aziz
parents:
diff changeset
106
239
7911f6a92e6e - Added 'new' and 'delete' declarations to Token and uncommented next and prev members. Added HEAD to TOK.
aziz
parents: 210
diff changeset
107 Token* next, prev;
131
ce636f3981cc - Removed TOK.Number.
aziz
parents: 107
diff changeset
108
4
92df59b1ec4a - Started implementation of scan().
aziz
parents: 3
diff changeset
109 char* start;
92df59b1ec4a - Started implementation of scan().
aziz
parents: 3
diff changeset
110 char* end;
92df59b1ec4a - Started implementation of scan().
aziz
parents: 3
diff changeset
111
0
8ba2570de175 Initial import.
aziz
parents:
diff changeset
112 union
8ba2570de175 Initial import.
aziz
parents:
diff changeset
113 {
31
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix.
aziz
parents: 30
diff changeset
114 struct
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix.
aziz
parents: 30
diff changeset
115 {
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix.
aziz
parents: 30
diff changeset
116 string str;
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix.
aziz
parents: 30
diff changeset
117 char pf;
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix.
aziz
parents: 30
diff changeset
118 }
62
96af5653acef - Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents: 58
diff changeset
119 dchar dchar_;
96af5653acef - Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents: 58
diff changeset
120 long long_;
96af5653acef - Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents: 58
diff changeset
121 ulong ulong_;
96af5653acef - Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents: 58
diff changeset
122 int int_;
96af5653acef - Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents: 58
diff changeset
123 uint uint_;
96af5653acef - Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents: 58
diff changeset
124 float float_;
96af5653acef - Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents: 58
diff changeset
125 double double_;
96af5653acef - Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents: 58
diff changeset
126 real real_;
0
8ba2570de175 Initial import.
aziz
parents:
diff changeset
127 }
28
3a9daccf7d96 - Added table for identifiers to Lexer.
aziz
parents: 27
diff changeset
128
107
722c05bbd5eb - Implemented parseEnumDeclaration() and added class EnumDeclaration.
aziz
parents: 97
diff changeset
129 alias srcText identifier;
722c05bbd5eb - Implemented parseEnumDeclaration() and added class EnumDeclaration.
aziz
parents: 97
diff changeset
130
65
6c21ae79fbb3 - Renamed function Token.span to Token.srcText.
aziz
parents: 62
diff changeset
131 string srcText()
28
3a9daccf7d96 - Added table for identifiers to Lexer.
aziz
parents: 27
diff changeset
132 {
30
426767b94635 - Added code for parsing the '#line' special token.
aziz
parents: 29
diff changeset
133 assert(start && end);
28
3a9daccf7d96 - Added table for identifiers to Lexer.
aziz
parents: 27
diff changeset
134 return start[0 .. end - start];
3a9daccf7d96 - Added table for identifiers to Lexer.
aziz
parents: 27
diff changeset
135 }
3a9daccf7d96 - Added table for identifiers to Lexer.
aziz
parents: 27
diff changeset
136
210
b7bde6583d3e - Made toString() static.
aziz
parents: 208
diff changeset
137 static string toString(TOK tok)
208
0a9bccf74046 - Added string table and toString() method to Token.
aziz
parents: 163
diff changeset
138 {
0a9bccf74046 - Added string table and toString() method to Token.
aziz
parents: 163
diff changeset
139 return tokToString[tok];
0a9bccf74046 - Added string table and toString() method to Token.
aziz
parents: 163
diff changeset
140 }
0a9bccf74046 - Added string table and toString() method to Token.
aziz
parents: 163
diff changeset
141
28
3a9daccf7d96 - Added table for identifiers to Lexer.
aziz
parents: 27
diff changeset
142 bool isKeyword()
3a9daccf7d96 - Added table for identifiers to Lexer.
aziz
parents: 27
diff changeset
143 {
163
f27a98bb17c7 - Fix: when parsing Declarator fails, type and ident is set to null.
aziz
parents: 131
diff changeset
144 return KeywordsBegin <= type && type <= KeywordsEnd;
28
3a9daccf7d96 - Added table for identifiers to Lexer.
aziz
parents: 27
diff changeset
145 }
208
0a9bccf74046 - Added string table and toString() method to Token.
aziz
parents: 163
diff changeset
146
163
f27a98bb17c7 - Fix: when parsing Declarator fails, type and ident is set to null.
aziz
parents: 131
diff changeset
147 int opEquals(TOK type2)
f27a98bb17c7 - Fix: when parsing Declarator fails, type and ident is set to null.
aziz
parents: 131
diff changeset
148 {
f27a98bb17c7 - Fix: when parsing Declarator fails, type and ident is set to null.
aziz
parents: 131
diff changeset
149 return type == type2;
f27a98bb17c7 - Fix: when parsing Declarator fails, type and ident is set to null.
aziz
parents: 131
diff changeset
150 }
239
7911f6a92e6e - Added 'new' and 'delete' declarations to Token and uncommented next and prev members. Added HEAD to TOK.
aziz
parents: 210
diff changeset
151
7911f6a92e6e - Added 'new' and 'delete' declarations to Token and uncommented next and prev members. Added HEAD to TOK.
aziz
parents: 210
diff changeset
152 new(size_t size)
7911f6a92e6e - Added 'new' and 'delete' declarations to Token and uncommented next and prev members. Added HEAD to TOK.
aziz
parents: 210
diff changeset
153 {
7911f6a92e6e - Added 'new' and 'delete' declarations to Token and uncommented next and prev members. Added HEAD to TOK.
aziz
parents: 210
diff changeset
154 void* p = malloc(size);
7911f6a92e6e - Added 'new' and 'delete' declarations to Token and uncommented next and prev members. Added HEAD to TOK.
aziz
parents: 210
diff changeset
155 if (p is null)
7911f6a92e6e - Added 'new' and 'delete' declarations to Token and uncommented next and prev members. Added HEAD to TOK.
aziz
parents: 210
diff changeset
156 throw new OutOfMemoryException();
7911f6a92e6e - Added 'new' and 'delete' declarations to Token and uncommented next and prev members. Added HEAD to TOK.
aziz
parents: 210
diff changeset
157 *cast(Token*)p = Token.init;
7911f6a92e6e - Added 'new' and 'delete' declarations to Token and uncommented next and prev members. Added HEAD to TOK.
aziz
parents: 210
diff changeset
158 return p;
7911f6a92e6e - Added 'new' and 'delete' declarations to Token and uncommented next and prev members. Added HEAD to TOK.
aziz
parents: 210
diff changeset
159 }
7911f6a92e6e - Added 'new' and 'delete' declarations to Token and uncommented next and prev members. Added HEAD to TOK.
aziz
parents: 210
diff changeset
160
7911f6a92e6e - Added 'new' and 'delete' declarations to Token and uncommented next and prev members. Added HEAD to TOK.
aziz
parents: 210
diff changeset
161 delete(void* p)
7911f6a92e6e - Added 'new' and 'delete' declarations to Token and uncommented next and prev members. Added HEAD to TOK.
aziz
parents: 210
diff changeset
162 {
7911f6a92e6e - Added 'new' and 'delete' declarations to Token and uncommented next and prev members. Added HEAD to TOK.
aziz
parents: 210
diff changeset
163 free(p);
7911f6a92e6e - Added 'new' and 'delete' declarations to Token and uncommented next and prev members. Added HEAD to TOK.
aziz
parents: 210
diff changeset
164 }
208
0a9bccf74046 - Added string table and toString() method to Token.
aziz
parents: 163
diff changeset
165 }
0a9bccf74046 - Added string table and toString() method to Token.
aziz
parents: 163
diff changeset
166
0a9bccf74046 - Added string table and toString() method to Token.
aziz
parents: 163
diff changeset
167 string[] tokToString = [
0a9bccf74046 - Added string table and toString() method to Token.
aziz
parents: 163
diff changeset
168 "Invalid",
0a9bccf74046 - Added string table and toString() method to Token.
aziz
parents: 163
diff changeset
169
0a9bccf74046 - Added string table and toString() method to Token.
aziz
parents: 163
diff changeset
170 "Identifier",
0a9bccf74046 - Added string table and toString() method to Token.
aziz
parents: 163
diff changeset
171 "Comment",
0a9bccf74046 - Added string table and toString() method to Token.
aziz
parents: 163
diff changeset
172 "String",
312
fa0b6f32c1ae - Added Special to enum TOK.
aziz
parents: 269
diff changeset
173 "Special",
208
0a9bccf74046 - Added string table and toString() method to Token.
aziz
parents: 163
diff changeset
174 "CharLiteral", "WCharLiteral", "DCharLiteral",
0a9bccf74046 - Added string table and toString() method to Token.
aziz
parents: 163
diff changeset
175
0a9bccf74046 - Added string table and toString() method to Token.
aziz
parents: 163
diff changeset
176 "Int32", "Int64", "Uint32", "Uint64",
0a9bccf74046 - Added string table and toString() method to Token.
aziz
parents: 163
diff changeset
177 "Float32", "Float64", "Float80",
0a9bccf74046 - Added string table and toString() method to Token.
aziz
parents: 163
diff changeset
178 "Imaginary32", "Imaginary64", "Imaginary80",
0a9bccf74046 - Added string table and toString() method to Token.
aziz
parents: 163
diff changeset
179
0a9bccf74046 - Added string table and toString() method to Token.
aziz
parents: 163
diff changeset
180 "(",
0a9bccf74046 - Added string table and toString() method to Token.
aziz
parents: 163
diff changeset
181 ")",
0a9bccf74046 - Added string table and toString() method to Token.
aziz
parents: 163
diff changeset
182 "[",
0a9bccf74046 - Added string table and toString() method to Token.
aziz
parents: 163
diff changeset
183 "]",
0a9bccf74046 - Added string table and toString() method to Token.
aziz
parents: 163
diff changeset
184 "{",
0a9bccf74046 - Added string table and toString() method to Token.
aziz
parents: 163
diff changeset
185 "}",
0a9bccf74046 - Added string table and toString() method to Token.
aziz
parents: 163
diff changeset
186
0a9bccf74046 - Added string table and toString() method to Token.
aziz
parents: 163
diff changeset
187 ".", "..", "...",
0a9bccf74046 - Added string table and toString() method to Token.
aziz
parents: 163
diff changeset
188
0a9bccf74046 - Added string table and toString() method to Token.
aziz
parents: 163
diff changeset
189 "Unordered",
0a9bccf74046 - Added string table and toString() method to Token.
aziz
parents: 163
diff changeset
190 "UorE",
0a9bccf74046 - Added string table and toString() method to Token.
aziz
parents: 163
diff changeset
191 "UorG",
0a9bccf74046 - Added string table and toString() method to Token.
aziz
parents: 163
diff changeset
192 "UorGorE",
0a9bccf74046 - Added string table and toString() method to Token.
aziz
parents: 163
diff changeset
193 "UorL",
0a9bccf74046 - Added string table and toString() method to Token.
aziz
parents: 163
diff changeset
194 "UorLorE",
0a9bccf74046 - Added string table and toString() method to Token.
aziz
parents: 163
diff changeset
195 "LorEorG",
0a9bccf74046 - Added string table and toString() method to Token.
aziz
parents: 163
diff changeset
196 "LorG",
0a9bccf74046 - Added string table and toString() method to Token.
aziz
parents: 163
diff changeset
197
0a9bccf74046 - Added string table and toString() method to Token.
aziz
parents: 163
diff changeset
198 "=", "==", "!=", "!",
0a9bccf74046 - Added string table and toString() method to Token.
aziz
parents: 163
diff changeset
199 "<=", "<",
0a9bccf74046 - Added string table and toString() method to Token.
aziz
parents: 163
diff changeset
200 ">=", ">",
0a9bccf74046 - Added string table and toString() method to Token.
aziz
parents: 163
diff changeset
201 "<<=", "<<",
0a9bccf74046 - Added string table and toString() method to Token.
aziz
parents: 163
diff changeset
202 ">>=",">>",
0a9bccf74046 - Added string table and toString() method to Token.
aziz
parents: 163
diff changeset
203 ">>>=", ">>>",
0a9bccf74046 - Added string table and toString() method to Token.
aziz
parents: 163
diff changeset
204 "|=", "||", "|",
0a9bccf74046 - Added string table and toString() method to Token.
aziz
parents: 163
diff changeset
205 "&=", "&&", "&",
0a9bccf74046 - Added string table and toString() method to Token.
aziz
parents: 163
diff changeset
206 "+=", "++", "+",
0a9bccf74046 - Added string table and toString() method to Token.
aziz
parents: 163
diff changeset
207 "-=", "--", "-",
0a9bccf74046 - Added string table and toString() method to Token.
aziz
parents: 163
diff changeset
208 "/=", "/",
0a9bccf74046 - Added string table and toString() method to Token.
aziz
parents: 163
diff changeset
209 "*=", "*",
0a9bccf74046 - Added string table and toString() method to Token.
aziz
parents: 163
diff changeset
210 "%=", "%",
0a9bccf74046 - Added string table and toString() method to Token.
aziz
parents: 163
diff changeset
211 "^=", "^",
0a9bccf74046 - Added string table and toString() method to Token.
aziz
parents: 163
diff changeset
212 "~=", "~",
0a9bccf74046 - Added string table and toString() method to Token.
aziz
parents: 163
diff changeset
213 "~",
0a9bccf74046 - Added string table and toString() method to Token.
aziz
parents: 163
diff changeset
214 "is", "!is",
0a9bccf74046 - Added string table and toString() method to Token.
aziz
parents: 163
diff changeset
215
0a9bccf74046 - Added string table and toString() method to Token.
aziz
parents: 163
diff changeset
216 ":",
0a9bccf74046 - Added string table and toString() method to Token.
aziz
parents: 163
diff changeset
217 ";",
0a9bccf74046 - Added string table and toString() method to Token.
aziz
parents: 163
diff changeset
218 "?",
0a9bccf74046 - Added string table and toString() method to Token.
aziz
parents: 163
diff changeset
219 ",",
0a9bccf74046 - Added string table and toString() method to Token.
aziz
parents: 163
diff changeset
220 "$",
0a9bccf74046 - Added string table and toString() method to Token.
aziz
parents: 163
diff changeset
221
0a9bccf74046 - Added string table and toString() method to Token.
aziz
parents: 163
diff changeset
222 "abstract","alias","align","asm","assert","auto","body",
0a9bccf74046 - Added string table and toString() method to Token.
aziz
parents: 163
diff changeset
223 "bool","break","byte","case","cast","catch","cdouble",
0a9bccf74046 - Added string table and toString() method to Token.
aziz
parents: 163
diff changeset
224 "cent","cfloat","char","class","const","continue","creal",
0a9bccf74046 - Added string table and toString() method to Token.
aziz
parents: 163
diff changeset
225 "dchar","debug","default","delegate","delete","deprecated","do",
0a9bccf74046 - Added string table and toString() method to Token.
aziz
parents: 163
diff changeset
226 "double","else","enum","export","extern","false","final",
0a9bccf74046 - Added string table and toString() method to Token.
aziz
parents: 163
diff changeset
227 "finally","float","for","foreach","foreach_reverse","function","goto",
0a9bccf74046 - Added string table and toString() method to Token.
aziz
parents: 163
diff changeset
228 "idouble","if","ifloat","import","in","inout","int",
0a9bccf74046 - Added string table and toString() method to Token.
aziz
parents: 163
diff changeset
229 "interface","invariant","ireal","is","lazy","long","macro",
0a9bccf74046 - Added string table and toString() method to Token.
aziz
parents: 163
diff changeset
230 "mixin","module","new","null","out","override","package",
0a9bccf74046 - Added string table and toString() method to Token.
aziz
parents: 163
diff changeset
231 "pragma","private","protected","public","real","ref","return",
0a9bccf74046 - Added string table and toString() method to Token.
aziz
parents: 163
diff changeset
232 "scope","short","static","struct","super","switch","synchronized",
0a9bccf74046 - Added string table and toString() method to Token.
aziz
parents: 163
diff changeset
233 "template","this","throw","true","try","typedef","typeid",
0a9bccf74046 - Added string table and toString() method to Token.
aziz
parents: 163
diff changeset
234 "typeof","ubyte","ucent","uint","ulong","union","unittest",
0a9bccf74046 - Added string table and toString() method to Token.
aziz
parents: 163
diff changeset
235 "ushort","version","void","volatile","wchar","while","with",
0a9bccf74046 - Added string table and toString() method to Token.
aziz
parents: 163
diff changeset
236
0a9bccf74046 - Added string table and toString() method to Token.
aziz
parents: 163
diff changeset
237 "EOF"
0a9bccf74046 - Added string table and toString() method to Token.
aziz
parents: 163
diff changeset
238 ];