Mercurial > projects > dil

comparison trunk/src/Token.d @ 314:ebd21bbf296e

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
- Added Whitespace, Sheband and Hashline to enum TOK. TOK.Whitespace is a flag and tokens that are considered whitespace are flagged as such. - Added method isWhitespace() to Token. Adapted parser accordingly.
author aziz
date Wed, 15 Aug 2007 19:57:01 +0000
parents fa0b6f32c1ae
children 6259fb93e3dd
comparison
equal deleted inserted replaced
313:1c1adededd8f 314:ebd21bbf296e
10 { 10 {
11 size_t loc; 11 size_t loc;
12 size_t col; 12 size_t col;
13 } 13 }
14 14
15 enum TOK 15 enum TOK : ushort
16 { 16 {
17 Invalid, 17 Invalid,
18 18
19 Identifier, 19 /// Flag for whitespace tokens that must be ignored in the parsing phase.
20 Comment, 20 Whitespace = 0x8000,
21 Comment = 1 | Whitespace,
22 Shebang = 2 | Whitespace,
23 HashLine = 3 | Whitespace,
24
25 Identifier = 4,
21 String, 26 String,
22 Special, 27 Special,
23 CharLiteral, WCharLiteral, DCharLiteral, 28 CharLiteral, WCharLiteral, DCharLiteral,
24 29
25 // Numbers 30 // Numbers
142 bool isKeyword() 147 bool isKeyword()
143 { 148 {
144 return KeywordsBegin <= type && type <= KeywordsEnd; 149 return KeywordsBegin <= type && type <= KeywordsEnd;
145 } 150 }
146 151
152 bool isWhitespace()
153 {
154 return !!(type & TOK.Whitespace);
155 }
156
147 int opEquals(TOK type2) 157 int opEquals(TOK type2)
148 { 158 {
149 return type == type2; 159 return type == type2;
150 } 160 }
151 161
165 } 175 }
166 176
167 string[] tokToString = [ 177 string[] tokToString = [
168 "Invalid", 178 "Invalid",
169 179
180 "Comment",
181 "#! /shebang/",
182 "#line",
183
170 "Identifier", 184 "Identifier",
171 "Comment",
172 "String", 185 "String",
173 "Special", 186 "Special",
174 "CharLiteral", "WCharLiteral", "DCharLiteral", 187 "CharLiteral", "WCharLiteral", "DCharLiteral",
175 188
176 "Int32", "Int64", "Uint32", "Uint64", 189 "Int32", "Int64", "Uint32", "Uint64",