Mercurial > projects > dil
annotate trunk/src/dil/Token.d @ 391:33b566df6af4
Migrated project to Tango.
Decremented the numbers of the format placeholders in the localized messages
by one.
Replaced all instances of writef/ln with Stdout.
Added module common.d with string aliases and a global Layout!(char) instance.
Replaced %s format specifiers with index placeholders in html/xml_tags.
Changed member Information.arguments to string message.
Copied std.metastring, std.uni and std.utf from Phobos.
author | Aziz K?ksal <aziz.koeksal@gmail.com> |
---|---|
date | Sat, 15 Sep 2007 17:12:26 +0200 |
parents | 4d36eea1bbc9 |
children | 4d9ee8e60712 |
rev | line source |
---|---|
0 | 1 /++ |
2 Author: Aziz Köksal | |
249 | 3 License: GPL3 |
0 | 4 +/ |
326 | 5 module dil.Token; |
391
33b566df6af4
Migrated project to Tango.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
390
diff
changeset
|
6 import common; |
33b566df6af4
Migrated project to Tango.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
390
diff
changeset
|
7 import tango.stdc.stdlib : malloc, free; |
33b566df6af4
Migrated project to Tango.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
390
diff
changeset
|
8 import tango.core.Exception; |
0 | 9 |
10 struct Position | |
11 { | |
12 size_t loc; | |
13 size_t col; | |
14 } | |
15 | |
314
ebd21bbf296e
- Added Whitespace, Sheband and Hashline to enum TOK. TOK.Whitespace is a flag and tokens that are considered whitespace are flagged as such.
aziz
parents:
312
diff
changeset
|
16 enum TOK : ushort |
3 | 17 { |
95 | 18 Invalid, |
19 | |
314
ebd21bbf296e
- Added Whitespace, Sheband and Hashline to enum TOK. TOK.Whitespace is a flag and tokens that are considered whitespace are flagged as such.
aziz
parents:
312
diff
changeset
|
20 /// Flag for whitespace tokens that must be ignored in the parsing phase. |
ebd21bbf296e
- Added Whitespace, Sheband and Hashline to enum TOK. TOK.Whitespace is a flag and tokens that are considered whitespace are flagged as such.
aziz
parents:
312
diff
changeset
|
21 Whitespace = 0x8000, |
390
4d36eea1bbc9
Refactored Lexer.scan().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
383
diff
changeset
|
22 Illegal = 1 | Whitespace, |
4d36eea1bbc9
Refactored Lexer.scan().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
383
diff
changeset
|
23 Comment = 2 | Whitespace, |
4d36eea1bbc9
Refactored Lexer.scan().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
383
diff
changeset
|
24 Shebang = 3 | Whitespace, |
4d36eea1bbc9
Refactored Lexer.scan().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
383
diff
changeset
|
25 HashLine = 4 | Whitespace, |
4d36eea1bbc9
Refactored Lexer.scan().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
383
diff
changeset
|
26 Filespec = 5 | Whitespace, |
314
ebd21bbf296e
- Added Whitespace, Sheband and Hashline to enum TOK. TOK.Whitespace is a flag and tokens that are considered whitespace are flagged as such.
aziz
parents:
312
diff
changeset
|
27 |
390
4d36eea1bbc9
Refactored Lexer.scan().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
383
diff
changeset
|
28 Identifier = 6, |
9
5d6968cc751e
- Parsing string and character literals now (rudimentary implementation.)
aziz
parents:
4
diff
changeset
|
29 String, |
82
fc645fb2fe72
- scanEscapeSequences() doesn't return 0xFFFF as an error value anymore, because it is a valid codepoint usable by the user.
aziz
parents:
71
diff
changeset
|
30 CharLiteral, WCharLiteral, DCharLiteral, |
58 | 31 |
343
95f1b6e43214
- Removed TOK.Special and added an own entry for each special token.
aziz
parents:
326
diff
changeset
|
32 // Special tokens |
95f1b6e43214
- Removed TOK.Special and added an own entry for each special token.
aziz
parents:
326
diff
changeset
|
33 FILE, |
95f1b6e43214
- Removed TOK.Special and added an own entry for each special token.
aziz
parents:
326
diff
changeset
|
34 LINE, |
95f1b6e43214
- Removed TOK.Special and added an own entry for each special token.
aziz
parents:
326
diff
changeset
|
35 DATE, |
95f1b6e43214
- Removed TOK.Special and added an own entry for each special token.
aziz
parents:
326
diff
changeset
|
36 TIME, |
95f1b6e43214
- Removed TOK.Special and added an own entry for each special token.
aziz
parents:
326
diff
changeset
|
37 TIMESTAMP, |
95f1b6e43214
- Removed TOK.Special and added an own entry for each special token.
aziz
parents:
326
diff
changeset
|
38 VENDOR, |
95f1b6e43214
- Removed TOK.Special and added an own entry for each special token.
aziz
parents:
326
diff
changeset
|
39 VERSION, |
95f1b6e43214
- Removed TOK.Special and added an own entry for each special token.
aziz
parents:
326
diff
changeset
|
40 |
359
511c14950cac
- Added messages MissingLinkageType and UnrecognizedLinkageType.
aziz
parents:
343
diff
changeset
|
41 // Number literals |
58 | 42 Int32, Int64, Uint32, Uint64, |
97
1a83e5070a84
- Added code for parsing IntNumber- and RealNumberExpressions.
aziz
parents:
95
diff
changeset
|
43 // Floating point number scanner relies on this order. (FloatXY + 3 == ImaginaryXY) |
62
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
58
diff
changeset
|
44 Float32, Float64, Float80, |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
58
diff
changeset
|
45 Imaginary32, Imaginary64, Imaginary80, |
58 | 46 |
36
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
47 |
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
48 // Brackets |
20 | 49 LParen, |
50 RParen, | |
51 LBracket, | |
52 RBracket, | |
53 LBrace, | |
54 RBrace, | |
21
c785c122e4e6
- Added code for parsing Colon, Semicolon, Question, Comma and Dollar.
aziz
parents:
20
diff
changeset
|
55 |
22
b05fff8e2ce4
- Added code for parsing Dot, Slice and Ellipses tokens.
aziz
parents:
21
diff
changeset
|
56 Dot, Slice, Ellipses, |
b05fff8e2ce4
- Added code for parsing Dot, Slice and Ellipses tokens.
aziz
parents:
21
diff
changeset
|
57 |
131 | 58 // Floating point number operators |
35
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
32
diff
changeset
|
59 Unordered, |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
32
diff
changeset
|
60 UorE, |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
32
diff
changeset
|
61 UorG, |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
32
diff
changeset
|
62 UorGorE, |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
32
diff
changeset
|
63 UorL, |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
32
diff
changeset
|
64 UorLorE, |
36
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
65 LorEorG, |
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
66 LorG, |
35
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
32
diff
changeset
|
67 |
36
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
68 // Normal operators |
35
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
32
diff
changeset
|
69 Assign, Equal, NotEqual, Not, |
38
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
36
diff
changeset
|
70 LessEqual, Less, |
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
36
diff
changeset
|
71 GreaterEqual, Greater, |
36
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
72 LShiftAssign, LShift, |
38
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
36
diff
changeset
|
73 RShiftAssign,RShift, |
640c45aaaaee
- Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents:
36
diff
changeset
|
74 URShiftAssign, URShift, |
23
1a7903701a3d
- Added code for parsing OrAssign, OrLogical and OrBinary tokens.
aziz
parents:
22
diff
changeset
|
75 OrAssign, OrLogical, OrBinary, |
24
903f91163f23
- Added code for parsing AndAssign, AndLogical and AndBinary tokens.
aziz
parents:
23
diff
changeset
|
76 AndAssign, AndLogical, AndBinary, |
25
9c866208b3f6
- Added code for parsing PlusAssign, PlusPlus, Plus, MinusAssign, MinusMinus, Minus tokens.
aziz
parents:
24
diff
changeset
|
77 PlusAssign, PlusPlus, Plus, |
9c866208b3f6
- Added code for parsing PlusAssign, PlusPlus, Plus, MinusAssign, MinusMinus, Minus tokens.
aziz
parents:
24
diff
changeset
|
78 MinusAssign, MinusMinus, Minus, |
32
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
79 DivAssign, Div, |
29
ef83eea26bbd
- Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents:
28
diff
changeset
|
80 MulAssign, Mul, |
ef83eea26bbd
- Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents:
28
diff
changeset
|
81 ModAssign, Mod, |
ef83eea26bbd
- Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents:
28
diff
changeset
|
82 XorAssign, Xor, |
27 | 83 CatAssign, Catenate, |
39
69b940398d7b
- Added unittest to test correct parsing of operator tokens.
aziz
parents:
38
diff
changeset
|
84 Tilde, |
71 | 85 Identity, NotIdentity, |
23
1a7903701a3d
- Added code for parsing OrAssign, OrLogical and OrBinary tokens.
aziz
parents:
22
diff
changeset
|
86 |
21
c785c122e4e6
- Added code for parsing Colon, Semicolon, Question, Comma and Dollar.
aziz
parents:
20
diff
changeset
|
87 Colon, |
c785c122e4e6
- Added code for parsing Colon, Semicolon, Question, Comma and Dollar.
aziz
parents:
20
diff
changeset
|
88 Semicolon, |
c785c122e4e6
- Added code for parsing Colon, Semicolon, Question, Comma and Dollar.
aziz
parents:
20
diff
changeset
|
89 Question, |
c785c122e4e6
- Added code for parsing Colon, Semicolon, Question, Comma and Dollar.
aziz
parents:
20
diff
changeset
|
90 Comma, |
c785c122e4e6
- Added code for parsing Colon, Semicolon, Question, Comma and Dollar.
aziz
parents:
20
diff
changeset
|
91 Dollar, |
c785c122e4e6
- Added code for parsing Colon, Semicolon, Question, Comma and Dollar.
aziz
parents:
20
diff
changeset
|
92 |
28 | 93 /* Keywords: |
94 NB.: Token.isKeyword() depends on this list being contiguous. | |
95 */ | |
96 Abstract,Alias,Align,Asm,Assert,Auto,Body, | |
97 Bool,Break,Byte,Case,Cast,Catch,Cdouble, | |
98 Cent,Cfloat,Char,Class,Const,Continue,Creal, | |
99 Dchar,Debug,Default,Delegate,Delete,Deprecated,Do, | |
100 Double,Else,Enum,Export,Extern,False,Final, | |
101 Finally,Float,For,Foreach,Foreach_reverse,Function,Goto, | |
102 Idouble,If,Ifloat,Import,In,Inout,Int, | |
269 | 103 Interface,Invariant,Ireal,Is,Lazy,Long,Macro/+D2.0+/, |
28 | 104 Mixin,Module,New,Null,Out,Override,Package, |
269 | 105 Pragma,Private,Protected,Public,Real,Ref/+D2.0+/,Return, |
28 | 106 Scope,Short,Static,Struct,Super,Switch,Synchronized, |
269 | 107 Template,This,Throw,Traits/+D2.0+/,True,Try,Typedef,Typeid, |
28 | 108 Typeof,Ubyte,Ucent,Uint,Ulong,Union,Unittest, |
109 Ushort,Version,Void,Volatile,Wchar,While,With, | |
110 | |
239
7911f6a92e6e
- Added 'new' and 'delete' declarations to Token and uncommented next and prev members. Added HEAD to TOK.
aziz
parents:
210
diff
changeset
|
111 HEAD, // start of linked list |
359
511c14950cac
- Added messages MissingLinkageType and UnrecognizedLinkageType.
aziz
parents:
343
diff
changeset
|
112 EOF, |
511c14950cac
- Added messages MissingLinkageType and UnrecognizedLinkageType.
aziz
parents:
343
diff
changeset
|
113 MAX |
3 | 114 } |
115 | |
28 | 116 alias TOK.Abstract KeywordsBegin; |
117 alias TOK.With KeywordsEnd; | |
118 | |
0 | 119 struct Token |
120 { | |
3 | 121 TOK type; |
84
ac8d961d10d1
- Added code for parsing This-,Super-,Null-,Bool-,Dollar-,CharLiteral- and StringLiteralExpression.
aziz
parents:
82
diff
changeset
|
122 // Position pos; |
0 | 123 |
239
7911f6a92e6e
- Added 'new' and 'delete' declarations to Token and uncommented next and prev members. Added HEAD to TOK.
aziz
parents:
210
diff
changeset
|
124 Token* next, prev; |
131 | 125 |
390
4d36eea1bbc9
Refactored Lexer.scan().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
383
diff
changeset
|
126 char* ws; /// Start of whitespace characters before token. Null if no WS. |
4d36eea1bbc9
Refactored Lexer.scan().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
383
diff
changeset
|
127 char* start; /// Start of token in source text. |
4d36eea1bbc9
Refactored Lexer.scan().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
383
diff
changeset
|
128 char* end; /// Points one past the end of token in source text. |
4 | 129 |
0 | 130 union |
131 { | |
31
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
132 struct |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
133 { |
323 | 134 Token* line_num; // #line number |
135 Token* line_filespec; // #line number filespec | |
136 } | |
137 struct | |
138 { | |
31
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
139 string str; |
383
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
372
diff
changeset
|
140 char pf; /// Postfix 'c', 'w' or 'd' |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
372
diff
changeset
|
141 version(D2) |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
372
diff
changeset
|
142 Token* tok_str; /// Points to the contents of a token string stored as a |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
372
diff
changeset
|
143 /// doubly linked list. The last token is always '}' or |
6a5fc22cae34
Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
372
diff
changeset
|
144 /// EOF in case end of source text is "q{" EOF. |
31
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
145 } |
62
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
58
diff
changeset
|
146 dchar dchar_; |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
58
diff
changeset
|
147 long long_; |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
58
diff
changeset
|
148 ulong ulong_; |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
58
diff
changeset
|
149 int int_; |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
58
diff
changeset
|
150 uint uint_; |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
58
diff
changeset
|
151 float float_; |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
58
diff
changeset
|
152 double double_; |
96af5653acef
- Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents:
58
diff
changeset
|
153 real real_; |
0 | 154 } |
28 | 155 |
107
722c05bbd5eb
- Implemented parseEnumDeclaration() and added class EnumDeclaration.
aziz
parents:
97
diff
changeset
|
156 alias srcText identifier; |
722c05bbd5eb
- Implemented parseEnumDeclaration() and added class EnumDeclaration.
aziz
parents:
97
diff
changeset
|
157 |
65 | 158 string srcText() |
28 | 159 { |
30 | 160 assert(start && end); |
28 | 161 return start[0 .. end - start]; |
162 } | |
163 | |
359
511c14950cac
- Added messages MissingLinkageType and UnrecognizedLinkageType.
aziz
parents:
343
diff
changeset
|
164 /// Find next non-whitespace token. Returns 'this' token if the next token is TOK.EOF or null. |
511c14950cac
- Added messages MissingLinkageType and UnrecognizedLinkageType.
aziz
parents:
343
diff
changeset
|
165 Token* nextNWS() |
511c14950cac
- Added messages MissingLinkageType and UnrecognizedLinkageType.
aziz
parents:
343
diff
changeset
|
166 out(token) |
511c14950cac
- Added messages MissingLinkageType and UnrecognizedLinkageType.
aziz
parents:
343
diff
changeset
|
167 { |
511c14950cac
- Added messages MissingLinkageType and UnrecognizedLinkageType.
aziz
parents:
343
diff
changeset
|
168 assert(token !is null); |
511c14950cac
- Added messages MissingLinkageType and UnrecognizedLinkageType.
aziz
parents:
343
diff
changeset
|
169 } |
511c14950cac
- Added messages MissingLinkageType and UnrecognizedLinkageType.
aziz
parents:
343
diff
changeset
|
170 body |
511c14950cac
- Added messages MissingLinkageType and UnrecognizedLinkageType.
aziz
parents:
343
diff
changeset
|
171 { |
511c14950cac
- Added messages MissingLinkageType and UnrecognizedLinkageType.
aziz
parents:
343
diff
changeset
|
172 auto token = next; |
511c14950cac
- Added messages MissingLinkageType and UnrecognizedLinkageType.
aziz
parents:
343
diff
changeset
|
173 while (token !is null && token.isWhitespace) |
511c14950cac
- Added messages MissingLinkageType and UnrecognizedLinkageType.
aziz
parents:
343
diff
changeset
|
174 token = token.next; |
511c14950cac
- Added messages MissingLinkageType and UnrecognizedLinkageType.
aziz
parents:
343
diff
changeset
|
175 if (token is null || token.type == TOK.EOF) |
511c14950cac
- Added messages MissingLinkageType and UnrecognizedLinkageType.
aziz
parents:
343
diff
changeset
|
176 return this; |
511c14950cac
- Added messages MissingLinkageType and UnrecognizedLinkageType.
aziz
parents:
343
diff
changeset
|
177 return token; |
511c14950cac
- Added messages MissingLinkageType and UnrecognizedLinkageType.
aziz
parents:
343
diff
changeset
|
178 } |
511c14950cac
- Added messages MissingLinkageType and UnrecognizedLinkageType.
aziz
parents:
343
diff
changeset
|
179 |
511c14950cac
- Added messages MissingLinkageType and UnrecognizedLinkageType.
aziz
parents:
343
diff
changeset
|
180 /// Find previous non-whitespace token. Returns 'this' token if the previous token is TOK.HEAD or null. |
511c14950cac
- Added messages MissingLinkageType and UnrecognizedLinkageType.
aziz
parents:
343
diff
changeset
|
181 Token* prevNWS() |
511c14950cac
- Added messages MissingLinkageType and UnrecognizedLinkageType.
aziz
parents:
343
diff
changeset
|
182 out(token) |
511c14950cac
- Added messages MissingLinkageType and UnrecognizedLinkageType.
aziz
parents:
343
diff
changeset
|
183 { |
511c14950cac
- Added messages MissingLinkageType and UnrecognizedLinkageType.
aziz
parents:
343
diff
changeset
|
184 assert(token !is null); |
511c14950cac
- Added messages MissingLinkageType and UnrecognizedLinkageType.
aziz
parents:
343
diff
changeset
|
185 } |
511c14950cac
- Added messages MissingLinkageType and UnrecognizedLinkageType.
aziz
parents:
343
diff
changeset
|
186 body |
511c14950cac
- Added messages MissingLinkageType and UnrecognizedLinkageType.
aziz
parents:
343
diff
changeset
|
187 { |
511c14950cac
- Added messages MissingLinkageType and UnrecognizedLinkageType.
aziz
parents:
343
diff
changeset
|
188 auto token = prev; |
511c14950cac
- Added messages MissingLinkageType and UnrecognizedLinkageType.
aziz
parents:
343
diff
changeset
|
189 while (token !is null && token.isWhitespace) |
511c14950cac
- Added messages MissingLinkageType and UnrecognizedLinkageType.
aziz
parents:
343
diff
changeset
|
190 token = token.prev; |
511c14950cac
- Added messages MissingLinkageType and UnrecognizedLinkageType.
aziz
parents:
343
diff
changeset
|
191 if (token is null || token.type == TOK.HEAD) |
511c14950cac
- Added messages MissingLinkageType and UnrecognizedLinkageType.
aziz
parents:
343
diff
changeset
|
192 return this; |
511c14950cac
- Added messages MissingLinkageType and UnrecognizedLinkageType.
aziz
parents:
343
diff
changeset
|
193 return token; |
511c14950cac
- Added messages MissingLinkageType and UnrecognizedLinkageType.
aziz
parents:
343
diff
changeset
|
194 } |
511c14950cac
- Added messages MissingLinkageType and UnrecognizedLinkageType.
aziz
parents:
343
diff
changeset
|
195 |
210 | 196 static string toString(TOK tok) |
208 | 197 { |
198 return tokToString[tok]; | |
199 } | |
200 | |
28 | 201 bool isKeyword() |
202 { | |
163
f27a98bb17c7
- Fix: when parsing Declarator fails, type and ident is set to null.
aziz
parents:
131
diff
changeset
|
203 return KeywordsBegin <= type && type <= KeywordsEnd; |
28 | 204 } |
208 | 205 |
314
ebd21bbf296e
- Added Whitespace, Sheband and Hashline to enum TOK. TOK.Whitespace is a flag and tokens that are considered whitespace are flagged as such.
aziz
parents:
312
diff
changeset
|
206 bool isWhitespace() |
ebd21bbf296e
- Added Whitespace, Sheband and Hashline to enum TOK. TOK.Whitespace is a flag and tokens that are considered whitespace are flagged as such.
aziz
parents:
312
diff
changeset
|
207 { |
ebd21bbf296e
- Added Whitespace, Sheband and Hashline to enum TOK. TOK.Whitespace is a flag and tokens that are considered whitespace are flagged as such.
aziz
parents:
312
diff
changeset
|
208 return !!(type & TOK.Whitespace); |
ebd21bbf296e
- Added Whitespace, Sheband and Hashline to enum TOK. TOK.Whitespace is a flag and tokens that are considered whitespace are flagged as such.
aziz
parents:
312
diff
changeset
|
209 } |
ebd21bbf296e
- Added Whitespace, Sheband and Hashline to enum TOK. TOK.Whitespace is a flag and tokens that are considered whitespace are flagged as such.
aziz
parents:
312
diff
changeset
|
210 |
343
95f1b6e43214
- Removed TOK.Special and added an own entry for each special token.
aziz
parents:
326
diff
changeset
|
211 bool isSpecialToken() |
95f1b6e43214
- Removed TOK.Special and added an own entry for each special token.
aziz
parents:
326
diff
changeset
|
212 { |
95f1b6e43214
- Removed TOK.Special and added an own entry for each special token.
aziz
parents:
326
diff
changeset
|
213 return *start == '_' && type != TOK.Identifier; |
95f1b6e43214
- Removed TOK.Special and added an own entry for each special token.
aziz
parents:
326
diff
changeset
|
214 } |
95f1b6e43214
- Removed TOK.Special and added an own entry for each special token.
aziz
parents:
326
diff
changeset
|
215 |
163
f27a98bb17c7
- Fix: when parsing Declarator fails, type and ident is set to null.
aziz
parents:
131
diff
changeset
|
216 int opEquals(TOK type2) |
f27a98bb17c7
- Fix: when parsing Declarator fails, type and ident is set to null.
aziz
parents:
131
diff
changeset
|
217 { |
f27a98bb17c7
- Fix: when parsing Declarator fails, type and ident is set to null.
aziz
parents:
131
diff
changeset
|
218 return type == type2; |
f27a98bb17c7
- Fix: when parsing Declarator fails, type and ident is set to null.
aziz
parents:
131
diff
changeset
|
219 } |
239
7911f6a92e6e
- Added 'new' and 'delete' declarations to Token and uncommented next and prev members. Added HEAD to TOK.
aziz
parents:
210
diff
changeset
|
220 |
7911f6a92e6e
- Added 'new' and 'delete' declarations to Token and uncommented next and prev members. Added HEAD to TOK.
aziz
parents:
210
diff
changeset
|
221 new(size_t size) |
7911f6a92e6e
- Added 'new' and 'delete' declarations to Token and uncommented next and prev members. Added HEAD to TOK.
aziz
parents:
210
diff
changeset
|
222 { |
7911f6a92e6e
- Added 'new' and 'delete' declarations to Token and uncommented next and prev members. Added HEAD to TOK.
aziz
parents:
210
diff
changeset
|
223 void* p = malloc(size); |
7911f6a92e6e
- Added 'new' and 'delete' declarations to Token and uncommented next and prev members. Added HEAD to TOK.
aziz
parents:
210
diff
changeset
|
224 if (p is null) |
391
33b566df6af4
Migrated project to Tango.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
390
diff
changeset
|
225 throw new OutOfMemoryException(__FILE__, __LINE__); |
239
7911f6a92e6e
- Added 'new' and 'delete' declarations to Token and uncommented next and prev members. Added HEAD to TOK.
aziz
parents:
210
diff
changeset
|
226 *cast(Token*)p = Token.init; |
7911f6a92e6e
- Added 'new' and 'delete' declarations to Token and uncommented next and prev members. Added HEAD to TOK.
aziz
parents:
210
diff
changeset
|
227 return p; |
7911f6a92e6e
- Added 'new' and 'delete' declarations to Token and uncommented next and prev members. Added HEAD to TOK.
aziz
parents:
210
diff
changeset
|
228 } |
7911f6a92e6e
- Added 'new' and 'delete' declarations to Token and uncommented next and prev members. Added HEAD to TOK.
aziz
parents:
210
diff
changeset
|
229 |
7911f6a92e6e
- Added 'new' and 'delete' declarations to Token and uncommented next and prev members. Added HEAD to TOK.
aziz
parents:
210
diff
changeset
|
230 delete(void* p) |
7911f6a92e6e
- Added 'new' and 'delete' declarations to Token and uncommented next and prev members. Added HEAD to TOK.
aziz
parents:
210
diff
changeset
|
231 { |
7911f6a92e6e
- Added 'new' and 'delete' declarations to Token and uncommented next and prev members. Added HEAD to TOK.
aziz
parents:
210
diff
changeset
|
232 free(p); |
7911f6a92e6e
- Added 'new' and 'delete' declarations to Token and uncommented next and prev members. Added HEAD to TOK.
aziz
parents:
210
diff
changeset
|
233 } |
208 | 234 } |
235 | |
359
511c14950cac
- Added messages MissingLinkageType and UnrecognizedLinkageType.
aziz
parents:
343
diff
changeset
|
236 const string[] tokToString = [ |
208 | 237 "Invalid", |
238 | |
390
4d36eea1bbc9
Refactored Lexer.scan().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents:
383
diff
changeset
|
239 "Illegal", |
314
ebd21bbf296e
- Added Whitespace, Sheband and Hashline to enum TOK. TOK.Whitespace is a flag and tokens that are considered whitespace are flagged as such.
aziz
parents:
312
diff
changeset
|
240 "Comment", |
ebd21bbf296e
- Added Whitespace, Sheband and Hashline to enum TOK. TOK.Whitespace is a flag and tokens that are considered whitespace are flagged as such.
aziz
parents:
312
diff
changeset
|
241 "#! /shebang/", |
ebd21bbf296e
- Added Whitespace, Sheband and Hashline to enum TOK. TOK.Whitespace is a flag and tokens that are considered whitespace are flagged as such.
aziz
parents:
312
diff
changeset
|
242 "#line", |
359
511c14950cac
- Added messages MissingLinkageType and UnrecognizedLinkageType.
aziz
parents:
343
diff
changeset
|
243 `"filespec"`, |
314
ebd21bbf296e
- Added Whitespace, Sheband and Hashline to enum TOK. TOK.Whitespace is a flag and tokens that are considered whitespace are flagged as such.
aziz
parents:
312
diff
changeset
|
244 |
208 | 245 "Identifier", |
246 "String", | |
247 "CharLiteral", "WCharLiteral", "DCharLiteral", | |
248 | |
359
511c14950cac
- Added messages MissingLinkageType and UnrecognizedLinkageType.
aziz
parents:
343
diff
changeset
|
249 "__FILE__", |
511c14950cac
- Added messages MissingLinkageType and UnrecognizedLinkageType.
aziz
parents:
343
diff
changeset
|
250 "__LINE__", |
511c14950cac
- Added messages MissingLinkageType and UnrecognizedLinkageType.
aziz
parents:
343
diff
changeset
|
251 "__DATE__", |
511c14950cac
- Added messages MissingLinkageType and UnrecognizedLinkageType.
aziz
parents:
343
diff
changeset
|
252 "__TIME__", |
511c14950cac
- Added messages MissingLinkageType and UnrecognizedLinkageType.
aziz
parents:
343
diff
changeset
|
253 "__TIMESTAMP__", |
511c14950cac
- Added messages MissingLinkageType and UnrecognizedLinkageType.
aziz
parents:
343
diff
changeset
|
254 "__VENDOR__", |
511c14950cac
- Added messages MissingLinkageType and UnrecognizedLinkageType.
aziz
parents:
343
diff
changeset
|
255 "__VERSION__", |
511c14950cac
- Added messages MissingLinkageType and UnrecognizedLinkageType.
aziz
parents:
343
diff
changeset
|
256 |
208 | 257 "Int32", "Int64", "Uint32", "Uint64", |
258 "Float32", "Float64", "Float80", | |
259 "Imaginary32", "Imaginary64", "Imaginary80", | |
260 | |
261 "(", | |
262 ")", | |
263 "[", | |
264 "]", | |
265 "{", | |
266 "}", | |
267 | |
268 ".", "..", "...", | |
269 | |
270 "Unordered", | |
271 "UorE", | |
272 "UorG", | |
273 "UorGorE", | |
274 "UorL", | |
275 "UorLorE", | |
276 "LorEorG", | |
277 "LorG", | |
278 | |
279 "=", "==", "!=", "!", | |
280 "<=", "<", | |
281 ">=", ">", | |
282 "<<=", "<<", | |
283 ">>=",">>", | |
284 ">>>=", ">>>", | |
285 "|=", "||", "|", | |
286 "&=", "&&", "&", | |
287 "+=", "++", "+", | |
288 "-=", "--", "-", | |
289 "/=", "/", | |
290 "*=", "*", | |
291 "%=", "%", | |
292 "^=", "^", | |
293 "~=", "~", | |
294 "~", | |
295 "is", "!is", | |
296 | |
297 ":", | |
298 ";", | |
299 "?", | |
300 ",", | |
301 "$", | |
302 | |
303 "abstract","alias","align","asm","assert","auto","body", | |
304 "bool","break","byte","case","cast","catch","cdouble", | |
305 "cent","cfloat","char","class","const","continue","creal", | |
306 "dchar","debug","default","delegate","delete","deprecated","do", | |
307 "double","else","enum","export","extern","false","final", | |
308 "finally","float","for","foreach","foreach_reverse","function","goto", | |
309 "idouble","if","ifloat","import","in","inout","int", | |
310 "interface","invariant","ireal","is","lazy","long","macro", | |
311 "mixin","module","new","null","out","override","package", | |
312 "pragma","private","protected","public","real","ref","return", | |
313 "scope","short","static","struct","super","switch","synchronized", | |
359
511c14950cac
- Added messages MissingLinkageType and UnrecognizedLinkageType.
aziz
parents:
343
diff
changeset
|
314 "template","this","throw","__traits","true","try","typedef","typeid", |
208 | 315 "typeof","ubyte","ucent","uint","ulong","union","unittest", |
316 "ushort","version","void","volatile","wchar","while","with", | |
317 | |
359
511c14950cac
- Added messages MissingLinkageType and UnrecognizedLinkageType.
aziz
parents:
343
diff
changeset
|
318 "HEAD", |
208 | 319 "EOF" |
320 ]; | |
359
511c14950cac
- Added messages MissingLinkageType and UnrecognizedLinkageType.
aziz
parents:
343
diff
changeset
|
321 static assert(tokToString.length == TOK.MAX); |