Mercurial > projects > dil

comparison trunk/src/dil/Token.d @ 552:3bc7801c207e

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Refactored the way how tokens are flagged as whitespace.
author Aziz K?ksal <aziz.koeksal@gmail.com>
date Thu, 20 Dec 2007 23:26:43 +0100
parents 943ecc9c133a
children 9e811db780a6
comparison
equal deleted inserted replaced
551:312da78ab301 552:3bc7801c207e
14 /++ 14 /++
15 A Token is a sequence of characters formed by the lexical analyzer. 15 A Token is a sequence of characters formed by the lexical analyzer.
16 +/ 16 +/
17 struct Token 17 struct Token
18 { 18 {
19 enum Flags : ushort
20 {
21 None,
22 Whitespace = 1, /// Tokens with this flag are ignored by the Parser.
23 }
24
19 TOK type; /// The type of the token. 25 TOK type; /// The type of the token.
26 Flags flags; /// The flags of the token.
20 /// Pointers to the next and previous tokens (doubly-linked list.) 27 /// Pointers to the next and previous tokens (doubly-linked list.)
21 Token* next, prev; 28 Token* next, prev;
22 29
23 char* ws; /// Start of whitespace characters before token. Null if no WS. 30 char* ws; /// Start of whitespace characters before token. Null if no WS.
24 char* start; /// Start of token in source text. 31 char* start; /// Start of token in source text.
111 static string toString(TOK tok) 118 static string toString(TOK tok)
112 { 119 {
113 return tokToString[tok]; 120 return tokToString[tok];
114 } 121 }
115 122
123 /// Adds Flags.Whitespace to this token's flags.
124 void setWhitespaceFlag()
125 {
126 this.flags |= Flags.Whitespace;
127 }
128
116 /++ 129 /++
117 Returns true if this is a token that can have newlines in it. 130 Returns true if this is a token that can have newlines in it.
118 These can be block and nested comments and any string literal 131 These can be block and nested comments and any string literal
119 except for escape string literals. 132 except for escape string literals.
120 +/ 133 +/
137 } 150 }
138 151
139 /// Returns true if this is a whitespace token. 152 /// Returns true if this is a whitespace token.
140 bool isWhitespace() 153 bool isWhitespace()
141 { 154 {
142 return !!(type & TOK.Whitespace); 155 return !!(flags & Flags.Whitespace);
143 } 156 }
144 157
145 /// Returns true if this is a special token. 158 /// Returns true if this is a special token.
146 bool isSpecialToken() 159 bool isSpecialToken()
147 { 160 {