# HG changeset patch # User aziz # Date 1185798964 0 # Node ID 7911f6a92e6ee36ea30b0420885ebf792fe73636 # Parent f3c6c15961bb6b249f1a3d2706046761d87a8e03 - Added 'new' and 'delete' declarations to Token and uncommented next and prev members. Added HEAD to TOK. - Tokens are stored in a doubly linked list by the Lexer now. - Added method scanNext() which can be used by peek() and nextToken(). - Saving 'token' in Parser.try_(). diff -r f3c6c15961bb -r 7911f6a92e6e trunk/src/Lexer.d --- a/trunk/src/Lexer.d Sat Jul 28 22:59:03 2007 +0000 +++ b/trunk/src/Lexer.d Mon Jul 30 12:36:04 2007 +0000 @@ -24,10 +24,11 @@ class Lexer { - Token token; + Token* head; /// The head of the doubly linked token list. + Token* token; /// Points to the current token in the token list. string text; - char* p; - char* end; + char* p; /// Points to the current character in the source text. + char* end; /// Points one character past the end of the source text. uint loc = 1; /// line of code @@ -35,7 +36,7 @@ Information[] errors; - bool reportErrors; +// bool reportErrors; Identifier[string] idtable; @@ -52,8 +53,12 @@ this.p = this.text.ptr; this.end = this.p + this.text.length; - this.reportErrors = true; +// this.reportErrors = true; loadKeywords(); + + this.head = new Token; + this.head.type = TOK.HEAD; + this.token = this.head; } public void scan(out Token t) @@ -1320,7 +1325,7 @@ ++p; MID mid; - Token t; + Token* t; uint oldloc = this.loc, newloc; peek(t); @@ -1434,7 +1439,7 @@ foreach(k; keywords) idtable[k.str] = k; } - +/+ struct State { Lexer lexer; @@ -1468,30 +1473,38 @@ { return State(this); } ++/ - void peek(ref Token t) + private void scanNext(ref Token* t) { - // Because peeked tokens are not stored in a linked - // list we need to switch off error reporting - // so as to avoid getting the same error more than once. - reportErrors = false; - char* save = p; - auto saveLoc = loc; - if (t.end !is null) // For successive peeks. + assert(t !is null); + if (t.next) + t = t.next; + else if (t.type != TOK.EOF) { - p = t.end; - assert(text.ptr < p && p <= end); + Token* new_t = new Token; + scan(*new_t); + new_t.prev = t; + t.next = new_t; + t = new_t; } - scan(t); - p = save; - loc = saveLoc; - reportErrors = true; + } + + void peek(ref Token* t) + { + scanNext(t); + } + + TOK nextToken() + { + scanNext(this.token); + return this.token.type; } void error(MID id, ...) { - if (reportErrors) - errors ~= new Information(InfoType.Lexer, id, loc, arguments(_arguments, _argptr)); +// if (reportErrors) + errors ~= new Information(InfoType.Lexer, id, loc, arguments(_arguments, _argptr)); } unittest @@ -1511,18 +1524,12 @@ writefln("end of peek() unittest"); } - public TOK nextToken() - { - scan(this.token); - return this.token.type; - } - Token[] getTokens() { Token[] tokens; - while (nextToken() != TOK.EOF) - tokens ~= this.token; - tokens ~= this.token; +// while (nextToken() != TOK.EOF) +// tokens ~= this.token; +// tokens ~= this.token; return tokens; } diff -r f3c6c15961bb -r 7911f6a92e6e trunk/src/Parser.d --- a/trunk/src/Parser.d Sat Jul 28 22:59:03 2007 +0000 +++ b/trunk/src/Parser.d Mon Jul 30 12:36:04 2007 +0000 @@ -25,7 +25,6 @@ this(char[] srcText, string fileName) { lx = new Lexer(srcText, fileName); - token = &lx.token; } char* prev; @@ -41,6 +40,7 @@ do { lx.nextToken(); + token = lx.token; if (!trying) { writef("\33[32m%s\33[0m", token.type); @@ -68,14 +68,17 @@ writef("\33[31mtry_\33[0m"); ++trying; // auto len = errors.length; + auto oldToken = token; auto oldCount = errorCount; - auto lexerState = lx.getState(); +// auto lexerState = lx.getState(); auto result = parseMethod(); // If the length of the array changed we know an error occurred. if (errorCount != oldCount) { - lexerState.restore(); // Restore state of the Lexer object. +// lexerState.restore(); // Restore state of the Lexer object. // errors = errors[0..len]; // Remove errors that were added when parseMethod() was called. + token = oldToken; + lx.token = oldToken; errorCount = oldCount; success = false; } @@ -88,7 +91,7 @@ TOK peekNext() { - Token next; + Token* next = token; lx.peek(next); return next.type; } @@ -1447,6 +1450,7 @@ switch (token.type) { case T.Align: + // TODO: don't call parseAttributeSpecifier(). d = parseAttributeSpecifier(); goto case_DeclarationStatement; /+ Not applicable for statements. @@ -1757,6 +1761,8 @@ nT(); s = new AttributeStatement(token.type, parse()); break; + // TODO: allow "scope class", "abstract scope class" in function bodies? + //case T.Class: default: s = new DeclarationStatement(parseDeclaration(stc)); } @@ -3086,7 +3092,7 @@ { // We count nested parentheses tokens because template types may appear inside parameter lists; e.g. (int x, Foo!(int) y). assert(token.type == T.LParen); - Token next; + Token* next = token; uint level = 1; while (1) { diff -r f3c6c15961bb -r 7911f6a92e6e trunk/src/Token.d --- a/trunk/src/Token.d Sat Jul 28 22:59:03 2007 +0000 +++ b/trunk/src/Token.d Mon Jul 30 12:36:04 2007 +0000 @@ -3,6 +3,8 @@ License: GPL2 +/ module Token; +import std.c.stdlib : malloc, free; +import std.outofmemory; struct Position { @@ -89,6 +91,7 @@ Typeof,Ubyte,Ucent,Uint,Ulong,Union,Unittest, Ushort,Version,Void,Volatile,Wchar,While,With, + HEAD, // start of linked list EOF } @@ -100,7 +103,7 @@ TOK type; // Position pos; -// Token* next, prev; + Token* next, prev; char* start; char* end; @@ -144,6 +147,20 @@ { return type == type2; } + + new(size_t size) + { + void* p = malloc(size); + if (p is null) + throw new OutOfMemoryException(); + *cast(Token*)p = Token.init; + return p; + } + + delete(void* p) + { + free(p); + } } string[] tokToString = [