Mercurial > projects > dil

comparison trunk/src/dil/lexer/Token.d @ 798:c24be8d4f6ab

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Added documentation comments.
author Aziz K?ksal <aziz.koeksal@gmail.com>
date Sat, 01 Mar 2008 02:53:06 +0100
parents 8380fb2c765f
children
comparison
equal deleted inserted replaced
797:cf2ad5df025c 798:c24be8d4f6ab
28 Token* next, prev; 28 Token* next, prev;
29 29
30 /// Start of whitespace characters before token. Null if no WS. 30 /// Start of whitespace characters before token. Null if no WS.
31 /// TODO: remove to save space; can be replaced by 'prev.end'. 31 /// TODO: remove to save space; can be replaced by 'prev.end'.
32 char* ws; 32 char* ws;
33 char* start; /// Start of token in source text. 33 char* start; /// Points to the first character of the token.
34 char* end; /// Points one past the end of token in source text. 34 char* end; /// Points one character past the end of the token.
35 35
36 /// Data associated with this token. 36 /// Data associated with this token.
37 /// TODO: move data structures out; use only pointers here to keep Token.sizeof small. 37 /// TODO: move data structures out; use only pointers here to keep Token.sizeof small.
38 union 38 union
39 { 39 {
43 struct 43 struct
44 { 44 {
45 Token* tokLineNum; /// #line number 45 Token* tokLineNum; /// #line number
46 Token* tokLineFilespec; /// #line number filespec 46 Token* tokLineFilespec; /// #line number filespec
47 } 47 }
48 /// For string tokens. 48 /// The value of a string token.
49 struct 49 struct
50 { 50 {
51 string str; /// Zero-terminated string. 51 string str; /// Zero-terminated string. (The zero is included in the length.)
52 char pf; /// Postfix 'c', 'w', 'd' or 0 for none. 52 char pf; /// Postfix 'c', 'w', 'd' or 0 for none.
53 version(D2) 53 version(D2)
54 Token* tok_str; /++ Points to the contents of a token string stored as a 54 Token* tok_str; /++ Points to the contents of a token string stored as a
55 doubly linked list. The last token is always '}' or 55 doubly linked list. The last token is always '}' or
56 EOF in case end of source text is "q{" EOF. 56 EOF in case end of source text is "q{" EOF.
57 +/ 57 +/
58 } 58 }
59 Identifier* ident; /// For keywords and identifiers. 59 Identifier* ident; /// For keywords and identifiers.
60 dchar dchar_; 60 dchar dchar_; /// A character value.
61 long long_; 61 long long_; /// A long integer value.
62 ulong ulong_; 62 ulong ulong_; /// An unsigned long integer value.
63 int int_; 63 int int_; /// An integer value.
64 uint uint_; 64 uint uint_; /// An unsigned integer value.
65 float float_; 65 float float_; /// A float value.
66 double double_; 66 double double_; /// A double value.
67 real real_; 67 real real_; /// A real value.
68 } 68 }
69 69
70 /// Returns the text of the token. 70 /// Returns the text of the token.
71 string srcText() 71 string srcText()
72 { 72 {
79 { 79 {
80 assert(ws && start); 80 assert(ws && start);
81 return ws[0 .. start - ws]; 81 return ws[0 .. start - ws];
82 } 82 }
83 83
84 /// Find next non-whitespace token. Returns 'this' token if the next token is TOK.EOF or null. 84 /// Finds the next non-whitespace token.
85 /// Returns: 'this' token if the next token is TOK.EOF or null.
85 Token* nextNWS() 86 Token* nextNWS()
86 out(token) 87 out(token)
87 { 88 {
88 assert(token !is null); 89 assert(token !is null);
89 } 90 }
95 if (token is null || token.kind == TOK.EOF) 96 if (token is null || token.kind == TOK.EOF)
96 return this; 97 return this;
97 return token; 98 return token;
98 } 99 }
99 100
100 /// Find previous non-whitespace token. Returns 'this' token if the previous token is TOK.HEAD or null. 101 /// Finds the previous non-whitespace token.
102 /// Returns: 'this' token if the previous token is TOK.HEAD or null.
101 Token* prevNWS() 103 Token* prevNWS()
102 out(token) 104 out(token)
103 { 105 {
104 assert(token !is null); 106 assert(token !is null);
105 } 107 }
111 if (token is null || token.kind == TOK.HEAD) 113 if (token is null || token.kind == TOK.HEAD)
112 return this; 114 return this;
113 return token; 115 return token;
114 } 116 }
115 117
116 /// Returns the string for token kind tok. 118 /// Returns the string for a token kind.
117 static string toString(TOK tok) 119 static string toString(TOK kind)
118 { 120 {
119 return tokToString[tok]; 121 return tokToString[kind];
120 } 122 }
121 123
122 /// Adds Flags. Whitespace to this token's flags. 124 /// Adds Flags.Whitespace to this.flags.
123 void setWhitespaceFlag() 125 void setWhitespaceFlag()
124 { 126 {
125 this.flags |= Flags.Whitespace; 127 this.flags |= Flags.Whitespace;
126 } 128 }
127 129
128 /// Returns true if this is a token that can have newlines in it. 130 /// Returns true if this is a token that can have newlines in it.
131 ///
129 /// These can be block and nested comments and any string literal 132 /// These can be block and nested comments and any string literal
130 /// except for escape string literals. 133 /// except for escape string literals.
131 bool isMultiline() 134 bool isMultiline()
132 { 135 {
133 return kind == TOK.String && start[0] != '\\' || 136 return kind == TOK.String && start[0] != '\\' ||
269 new(size_t size) 272 new(size_t size)
270 { 273 {
271 void* p = malloc(size); 274 void* p = malloc(size);
272 if (p is null) 275 if (p is null)
273 throw new OutOfMemoryException(__FILE__, __LINE__); 276 throw new OutOfMemoryException(__FILE__, __LINE__);
277 // TODO: Token.init should be all zeros.
278 // Maybe use calloc() to avoid this line?
274 *cast(Token*)p = Token.init; 279 *cast(Token*)p = Token.init;
275 return p; 280 return p;
276 } 281 }
277 282
278 /// Deletes a token using free(). 283 /// Deletes a token using free().