Mercurial > projects > dil

comparison trunk/src/dil/lexer/Token.d @ 783:8380fb2c765f

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Added documentation comments.
author Aziz K?ksal <aziz.koeksal@gmail.com>
date Sat, 23 Feb 2008 02:15:41 +0100
parents 6dbbb403fc58
children c24be8d4f6ab
comparison
equal deleted inserted replaced
782:c4b28cf38760 783:8380fb2c765f
11 import tango.core.Exception; 11 import tango.core.Exception;
12 import common; 12 import common;
13 13
14 public import dil.lexer.TokensEnum; 14 public import dil.lexer.TokensEnum;
15 15
16 /++ 16 /// A Token is a sequence of characters formed by the lexical analyzer.
17 A Token is a sequence of characters formed by the lexical analyzer.
18 +/
19 struct Token 17 struct Token
20 { 18 { /// Flags set by the Lexer.
21 enum Flags : ushort 19 enum Flags : ushort
22 { 20 {
23 None, 21 None,
24 Whitespace = 1, /// Tokens with this flag are ignored by the Parser. 22 Whitespace = 1, /// Tokens with this flag are ignored by the Parser.
25 } 23 }
27 TOK kind; /// The token kind. 25 TOK kind; /// The token kind.
28 Flags flags; /// The flags of the token. 26 Flags flags; /// The flags of the token.
29 /// Pointers to the next and previous tokens (doubly-linked list.) 27 /// Pointers to the next and previous tokens (doubly-linked list.)
30 Token* next, prev; 28 Token* next, prev;
31 29
32 char* ws; /// Start of whitespace characters before token. Null if no WS. 30 /// Start of whitespace characters before token. Null if no WS.
31 /// TODO: remove to save space; can be replaced by 'prev.end'.
32 char* ws;
33 char* start; /// Start of token in source text. 33 char* start; /// Start of token in source text.
34 char* end; /// Points one past the end of token in source text. 34 char* end; /// Points one past the end of token in source text.
35 35
36 // Data associated with this token. 36 /// Data associated with this token.
37 /// TODO: move data structures out; use only pointers here to keep Token.sizeof small.
37 union 38 union
38 { 39 {
39 /// For newline tokens. 40 /// For newline tokens.
40 NewlineData newline; 41 NewlineData newline;
41 /// For #line tokens. 42 /// For #line tokens.
45 Token* tokLineFilespec; /// #line number filespec 46 Token* tokLineFilespec; /// #line number filespec
46 } 47 }
47 /// For string tokens. 48 /// For string tokens.
48 struct 49 struct
49 { 50 {
50 string str; 51 string str; /// Zero-terminated string.
51 char pf; /// Postfix 'c', 'w', 'd' or 0 for none. 52 char pf; /// Postfix 'c', 'w', 'd' or 0 for none.
52 version(D2) 53 version(D2)
53 Token* tok_str; /++ Points to the contents of a token string stored as a 54 Token* tok_str; /++ Points to the contents of a token string stored as a
54 doubly linked list. The last token is always '}' or 55 doubly linked list. The last token is always '}' or
55 EOF in case end of source text is "q{" EOF. 56 EOF in case end of source text is "q{" EOF.
56 +/ 57 +/
57 } 58 }
58 Identifier* ident; 59 Identifier* ident; /// For keywords and identifiers.
59 dchar dchar_; 60 dchar dchar_;
60 long long_; 61 long long_;
61 ulong ulong_; 62 ulong ulong_;
62 int int_; 63 int int_;
63 uint uint_; 64 uint uint_;
64 float float_; 65 float float_;
65 double double_; 66 double double_;
66 real real_; 67 real real_;
67 } 68 }
68 69
69 alias srcText identifier;
70
71 /// Returns the text of the token. 70 /// Returns the text of the token.
72 string srcText() 71 string srcText()
73 { 72 {
74 assert(start && end); 73 assert(start && end);
75 return start[0 .. end - start]; 74 return start[0 .. end - start];
112 if (token is null || token.kind == TOK.HEAD) 111 if (token is null || token.kind == TOK.HEAD)
113 return this; 112 return this;
114 return token; 113 return token;
115 } 114 }
116 115
116 /// Returns the string for token kind tok.
117 static string toString(TOK tok) 117 static string toString(TOK tok)
118 { 118 {
119 return tokToString[tok]; 119 return tokToString[tok];
120 } 120 }
121 121
123 void setWhitespaceFlag() 123 void setWhitespaceFlag()
124 { 124 {
125 this.flags |= Flags.Whitespace; 125 this.flags |= Flags.Whitespace;
126 } 126 }
127 127
128 /++ 128 /// Returns true if this is a token that can have newlines in it.
129 Returns true if this is a token that can have newlines in it. 129 /// These can be block and nested comments and any string literal
130 These can be block and nested comments and any string literal 130 /// except for escape string literals.
131 except for escape string literals.
132 +/
133 bool isMultiline() 131 bool isMultiline()
134 { 132 {
135 return kind == TOK.String && start[0] != '\\' || 133 return kind == TOK.String && start[0] != '\\' ||
136 kind == TOK.Comment && start[1] != '/'; 134 kind == TOK.Comment && start[1] != '/';
137 } 135 }
265 { 263 {
266 assert(left.end <= right.start || left is right ); 264 assert(left.end <= right.start || left is right );
267 return left.start[0 .. right.end - left.start]; 265 return left.start[0 .. right.end - left.start];
268 } 266 }
269 267
268 /// Uses malloc() to allocate memory for a token.
270 new(size_t size) 269 new(size_t size)
271 { 270 {
272 void* p = malloc(size); 271 void* p = malloc(size);
273 if (p is null) 272 if (p is null)
274 throw new OutOfMemoryException(__FILE__, __LINE__); 273 throw new OutOfMemoryException(__FILE__, __LINE__);
275 *cast(Token*)p = Token.init; 274 *cast(Token*)p = Token.init;
276 return p; 275 return p;
277 } 276 }
278 277
278 /// Deletes a token using free().
279 delete(void* p) 279 delete(void* p)
280 { 280 {
281 auto token = cast(Token*)p; 281 auto token = cast(Token*)p;
282 if (token) 282 if (token)
283 { 283 {