Mercurial > projects > dil

comparison trunk/src/Token.d @ 239:7911f6a92e6e

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
- Added 'new' and 'delete' declarations to Token and uncommented next and prev members. Added HEAD to TOK. - Tokens are stored in a doubly linked list by the Lexer now. - Added method scanNext() which can be used by peek() and nextToken(). - Saving 'token' in Parser.try_().
author aziz
date Mon, 30 Jul 2007 12:36:04 +0000
parents b7bde6583d3e
children 32d354584b28
comparison
equal deleted inserted replaced
238:f3c6c15961bb 239:7911f6a92e6e
1 /++ 1 /++
2 Author: Aziz Köksal 2 Author: Aziz Köksal
3 License: GPL2 3 License: GPL2
4 +/ 4 +/
5 module Token; 5 module Token;
6 import std.c.stdlib : malloc, free;
7 import std.outofmemory;
6 8
7 struct Position 9 struct Position
8 { 10 {
9 size_t loc; 11 size_t loc;
10 size_t col; 12 size_t col;
87 Scope,Short,Static,Struct,Super,Switch,Synchronized, 89 Scope,Short,Static,Struct,Super,Switch,Synchronized,
88 Template,This,Throw,True,Try,Typedef,Typeid, 90 Template,This,Throw,True,Try,Typedef,Typeid,
89 Typeof,Ubyte,Ucent,Uint,Ulong,Union,Unittest, 91 Typeof,Ubyte,Ucent,Uint,Ulong,Union,Unittest,
90 Ushort,Version,Void,Volatile,Wchar,While,With, 92 Ushort,Version,Void,Volatile,Wchar,While,With,
91 93
94 HEAD, // start of linked list
92 EOF 95 EOF
93 } 96 }
94 97
95 alias TOK.Abstract KeywordsBegin; 98 alias TOK.Abstract KeywordsBegin;
96 alias TOK.With KeywordsEnd; 99 alias TOK.With KeywordsEnd;
98 struct Token 101 struct Token
99 { 102 {
100 TOK type; 103 TOK type;
101 // Position pos; 104 // Position pos;
102 105
103 // Token* next, prev; 106 Token* next, prev;
104 107
105 char* start; 108 char* start;
106 char* end; 109 char* end;
107 110
108 union 111 union
141 } 144 }
142 145
143 int opEquals(TOK type2) 146 int opEquals(TOK type2)
144 { 147 {
145 return type == type2; 148 return type == type2;
149 }
150
151 new(size_t size)
152 {
153 void* p = malloc(size);
154 if (p is null)
155 throw new OutOfMemoryException();
156 *cast(Token*)p = Token.init;
157 return p;
158 }
159
160 delete(void* p)
161 {
162 free(p);
146 } 163 }
147 } 164 }
148 165
149 string[] tokToString = [ 166 string[] tokToString = [
150 "Invalid", 167 "Invalid",