# HG changeset patch # User Aziz K?ksal # Date 1197379170 -3600 # Node ID 4e14cd1b24dadc106f76624d96fbc7bf90d4bc46 # Parent 949a53332c6640bf52af7257d08275f5077120c2 Refactored code and added modules related to tabulated Identifiers. Rearranged members of struct Identifier and added new member ID identID. Moved idTableLookup to module dil.IdTable. Renamed module TokenIDs to TokensEnum. Added member Identifier* ident to struct Token. Changed string switchtes in Parser to integer switches using enum ID. diff -r 949a53332c66 -r 4e14cd1b24da trunk/src/dil/Expressions.d --- a/trunk/src/dil/Expressions.d Sun Dec 09 23:27:40 2007 +0100 +++ b/trunk/src/dil/Expressions.d Tue Dec 11 14:19:30 2007 +0100 @@ -8,6 +8,7 @@ import dil.Types; import dil.Declarations; import dil.Statements; +import dil.Identifier; import dil.Scope; abstract class Expression : Node @@ -1012,9 +1013,9 @@ class AsmRegisterExpression : Expression { - Token* register; - Token* number; // ST(0) - ST(7) or FS:0, FS:4, FS:8 - this(Token* register, Token* number = null) + Identifier* register; + int number; // ST(0) - ST(7) or FS:0, FS:4, FS:8 + this(Identifier* register, int number = -1) { mixin(set_kind); this.register = register; diff -r 949a53332c66 -r 4e14cd1b24da trunk/src/dil/IdTable.d --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/trunk/src/dil/IdTable.d Tue Dec 11 14:19:30 2007 +0100 @@ -0,0 +1,99 @@ +/++ + Author: Aziz Köksal + License: GPL3 ++/ +module dil.IdTable; +import dil.Identifier; +import dil.IdentsEnum; +import dil.TokensEnum; +import dil.IdentsGenerator; +import dil.Keywords; +import common; + +struct Ident +{ + const static + { + mixin(generateIdentMembers()); + } + + static Identifier*[] allIds() + { + return __allIds; + } +} + +struct IdTable +{ +static: + /// A set of common, predefined identifiers for fast lookups. + private Identifier*[string] staticTable; + /// A table that grows with every newly found, unique identifier. + /// Access must be synchronized. + private Identifier*[string] growingTable; + + /// Initializes the static table. + static this() + { + foreach (ref k; keywords) + staticTable[k.str] = &k; + foreach (id; Ident.allIds()) + staticTable[id.str] = id; + staticTable.rehash; + } + + /// Looks in both tables. + Identifier* lookup(string idString) + { + auto id = inStatic(idString); + if (id) + return id; + return inGrowing(idString); + } + + /// Look up idString in the static table. + Identifier* inStatic(string idString) + { + auto id = idString in staticTable; + return id ? *id : null; + } + + /++ + Returns the Identifier for idString. + Adds idString to the table if not found. + Access to the data structure is synchronized. + +/ + Identifier* inGrowing(string idString) + out(id) + { assert(id !is null); } + body + { + synchronized + { + auto id = idString in growingTable; + if (id) + return *id; + auto newID = Identifier(idString, TOK.Identifier); + growingTable[idString] = newID; + return newID; + } + } + + /+ + Identifier* addIdentifiers(char[][] idStrings) + { + auto ids = new Identifier*[idStrings.length]; + foreach (i, idString; idStrings) + { + Identifier** id = idString in tabulatedIds; + if (!id) + { + auto newID = Identifier(TOK.Identifier, idString); + tabulatedIds[idString] = newID; + id = &newID; + } + ids[i] = *id; + } + } + +/ +} diff -r 949a53332c66 -r 4e14cd1b24da trunk/src/dil/Identifier.d --- a/trunk/src/dil/Identifier.d Sun Dec 09 23:27:40 2007 +0100 +++ b/trunk/src/dil/Identifier.d Tue Dec 11 14:19:30 2007 +0100 @@ -3,20 +3,24 @@ License: GPL3 +/ module dil.Identifier; -import dil.Token; +import dil.TokensEnum; +import dil.IdentsEnum; import common; +align(1) struct Identifier { + string str; TOK type; - string str; + ID identID; - static Identifier* opCall(TOK type, string str) + static Identifier* opCall(string str, TOK type, ID identID = ID.Null) { - auto i = new Identifier; - i.type = type; - i.str = str; - return i; + auto id = new Identifier; + id.str = str; + id.type = type; + id.identID = identID; + return id; } uint toHash() @@ -29,3 +33,4 @@ return hash; } } +// pragma(msg, Identifier.sizeof.stringof); diff -r 949a53332c66 -r 4e14cd1b24da trunk/src/dil/IdentsEnum.d --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/trunk/src/dil/IdentsEnum.d Tue Dec 11 14:19:30 2007 +0100 @@ -0,0 +1,13 @@ +/++ + Author: Aziz Köksal + License: GPL3 ++/ +module dil.IdentsEnum; +import dil.IdentsGenerator; + +mixin( + "enum ID : ushort {" + "Null," + ~ generateIDMembers ~ + "}" +); diff -r 949a53332c66 -r 4e14cd1b24da trunk/src/dil/IdentsGenerator.d --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/trunk/src/dil/IdentsGenerator.d Tue Dec 11 14:19:30 2007 +0100 @@ -0,0 +1,93 @@ +/++ + Author: Aziz Köksal + License: GPL3 ++/ +module dil.IdentsGenerator; + +struct StrPair +{ +const: + char[] str; /// Identifier string in code. + char[] idStr; /// In table. +} + +static const StrPair[] identPairs = [ + // scope: + {"exit"}, + {"success"}, + {"failure"}, + // pragma: + {"msg"}, {"lib"}, + // Linkage: + {"C"}, {"D"}, {"Windows"}, {"Pascal"}, {"System"}, + // Operator methods: + {"opCall"}, + // ASM identifiers: + {"near"}, {"far"}, {"word"}, {"dword"}, {"qword"}, + {"ptr"}, {"offset"}, {"seg"}, {"__LOCAL_SIZE"}, + {"FS"}, {"ST"}, + {"AL"}, {"AH"}, {"AX"}, {"EAX"}, + {"BL"}, {"BH"}, {"BX"}, {"EBX"}, + {"CL"}, {"CH"}, {"CX"}, {"ECX"}, + {"DL"}, {"DH"}, {"DX"}, {"EDX"}, + {"BP"}, {"EBP"}, {"SP"}, {"ESP"}, + {"DI"}, {"EDI"}, {"SI"}, {"ESI"}, + {"ES"}, {"CS"}, {"SS"}, {"DS"}, {"GS"}, + {"CR0"}, {"CR2"}, {"CR3"}, {"CR4"}, + {"DR0"}, {"DR1"}, {"DR2"}, {"DR3"}, {"DR6"}, {"DR7"}, + {"TR3"}, {"TR4"}, {"TR5"}, {"TR6"}, {"TR7"}, + {"MM0"}, {"MM1"}, {"MM2"}, {"MM3"}, + {"MM4"}, {"MM5"}, {"MM6"}, {"MM7"}, + {"XMM0"}, {"XMM1"}, {"XMM2"}, {"XMM3"}, + {"XMM4"}, {"XMM5"}, {"XMM6"}, {"XMM7"}, +]; + +/++ + CTF for generating the members of the struct Ident. + The resulting string could look like this: + --- + private struct Ids {static const: + Identifier _str = {"str", TOK.Identifier, ID.str}; + // more ... + } + Identifier* str = &Ids._str; + // more ... + private Identifier*[] __allIds = [ + str, + // more ... + ] + --- ++/ +char[] generateIdentMembers(char[] private_members = "") +{ + private_members = "private struct Ids {static const:"; + + char[] public_members = ""; + char[] array = "private Identifier*[] __allIds = ["; + foreach (pair; identPairs) + { + // NB: conditional makes function uneligible for CTE. + // char[] idString = pair.idStr ? pair.idStr : pair.str; + // Identifier _str = {"str", TOK.Identifier, ID.str}; + private_members ~= "Identifier _"~pair.str~` = {"`~pair.str~`", TOK.Identifier, ID.`~pair.str~"};\n"; + // Identifier* str = &_str; + public_members ~= "Identifier* "~pair.str~" = &Ids._"~pair.str~";\n"; + array ~= pair.str~","; + } + + private_members ~= "}"; // Close private { + array ~= "];"; + + return private_members ~ public_members ~ array; +} + +/// CTF for generating the members of the enum ID. +char[] generateIDMembers(char[] members = "") +{ + foreach (pair; identPairs) + members ~= pair.str ~ ",\n"; + return members; +} + +// pragma(msg, generateIdentMembers()); +// pragma(msg, generateIDMembers()); diff -r 949a53332c66 -r 4e14cd1b24da trunk/src/dil/Keywords.d --- a/trunk/src/dil/Keywords.d Sun Dec 09 23:27:40 2007 +0100 +++ b/trunk/src/dil/Keywords.d Tue Dec 11 14:19:30 2007 +0100 @@ -8,112 +8,112 @@ /// Table of reserved identifiers. static const Identifier[] keywords = [ - {TOK.Abstract, "abstract"}, - {TOK.Alias, "alias"}, - {TOK.Align, "align"}, - {TOK.Asm, "asm"}, - {TOK.Assert, "assert"}, - {TOK.Auto, "auto"}, - {TOK.Body, "body"}, - {TOK.Bool, "bool"}, - {TOK.Break, "break"}, - {TOK.Byte, "byte"}, - {TOK.Case, "case"}, - {TOK.Cast, "cast"}, - {TOK.Catch, "catch"}, - {TOK.Cdouble, "cdouble"}, - {TOK.Cent, "cent"}, - {TOK.Cfloat, "cfloat"}, - {TOK.Char, "char"}, - {TOK.Class, "class"}, - {TOK.Const, "const"}, - {TOK.Continue, "continue"}, - {TOK.Creal, "creal"}, - {TOK.Dchar, "dchar"}, - {TOK.Debug, "debug"}, - {TOK.Default, "default"}, - {TOK.Delegate, "delegate"}, - {TOK.Delete, "delete"}, - {TOK.Deprecated, "deprecated"}, - {TOK.Do, "do"}, - {TOK.Double, "double"}, - {TOK.Else, "else"}, - {TOK.Enum, "enum"}, - {TOK.Export, "export"}, - {TOK.Extern, "extern"}, - {TOK.False, "false"}, - {TOK.Final, "final"}, - {TOK.Finally, "finally"}, - {TOK.Float, "float"}, - {TOK.For, "for"}, - {TOK.Foreach, "foreach"}, - {TOK.Foreach_reverse, "foreach_reverse"}, - {TOK.Function, "function"}, - {TOK.Goto, "goto"}, - {TOK.Idouble, "idouble"}, - {TOK.If, "if"}, - {TOK.Ifloat, "ifloat"}, - {TOK.Import, "import"}, - {TOK.In, "in"}, - {TOK.Inout, "inout"}, - {TOK.Int, "int"}, - {TOK.Interface, "interface"}, - {TOK.Invariant, "invariant"}, - {TOK.Ireal, "ireal"}, - {TOK.Is, "is"}, - {TOK.Lazy, "lazy"}, - {TOK.Long, "long"}, - {TOK.Macro, "macro"}, // D2.0 - {TOK.Mixin, "mixin"}, - {TOK.Module, "module"}, - {TOK.New, "new"}, - {TOK.Null, "null"}, - {TOK.Out, "out"}, - {TOK.Override, "override"}, - {TOK.Package, "package"}, - {TOK.Pragma, "pragma"}, - {TOK.Private, "private"}, - {TOK.Protected, "protected"}, - {TOK.Public, "public"}, - {TOK.Real, "real"}, - {TOK.Ref, "ref"}, - {TOK.Return, "return"}, - {TOK.Scope, "scope"}, - {TOK.Short, "short"}, - {TOK.Static, "static"}, - {TOK.Struct, "struct"}, - {TOK.Super, "super"}, - {TOK.Switch, "switch"}, - {TOK.Synchronized, "synchronized"}, - {TOK.Template, "template"}, - {TOK.This, "this"}, - {TOK.Throw, "throw"}, - {TOK.Traits, "__traits"}, // D2.0 - {TOK.True, "true"}, - {TOK.Try, "try"}, - {TOK.Typedef, "typedef"}, - {TOK.Typeid, "typeid"}, - {TOK.Typeof, "typeof"}, - {TOK.Ubyte, "ubyte"}, - {TOK.Ucent, "ucent"}, - {TOK.Uint, "uint"}, - {TOK.Ulong, "ulong"}, - {TOK.Union, "union"}, - {TOK.Unittest, "unittest"}, - {TOK.Ushort, "ushort"}, - {TOK.Version, "version"}, - {TOK.Void, "void"}, - {TOK.Volatile, "volatile"}, - {TOK.Wchar, "wchar"}, - {TOK.While, "while"}, - {TOK.With, "with"}, + {"abstract", TOK.Abstract}, + {"alias", TOK.Alias}, + {"align", TOK.Align}, + {"asm", TOK.Asm}, + {"assert", TOK.Assert}, + {"auto", TOK.Auto}, + {"body", TOK.Body}, + {"bool", TOK.Bool}, + {"break", TOK.Break}, + {"byte", TOK.Byte}, + {"case", TOK.Case}, + {"cast", TOK.Cast}, + {"catch", TOK.Catch}, + {"cdouble", TOK.Cdouble}, + {"cent", TOK.Cent}, + {"cfloat", TOK.Cfloat}, + {"char", TOK.Char}, + {"class", TOK.Class}, + {"const", TOK.Const}, + {"continue", TOK.Continue}, + {"creal", TOK.Creal}, + {"dchar", TOK.Dchar}, + {"debug", TOK.Debug}, + {"default", TOK.Default}, + {"delegate", TOK.Delegate}, + {"delete", TOK.Delete}, + {"deprecated", TOK.Deprecated}, + {"do", TOK.Do}, + {"double", TOK.Double}, + {"else", TOK.Else}, + {"enum", TOK.Enum}, + {"export", TOK.Export}, + {"extern", TOK.Extern}, + {"false", TOK.False}, + {"final", TOK.Final}, + {"finally", TOK.Finally}, + {"float", TOK.Float}, + {"for", TOK.For}, + {"foreach", TOK.Foreach}, + {"foreach_reverse", TOK.Foreach_reverse}, + {"function", TOK.Function}, + {"goto", TOK.Goto}, + {"idouble", TOK.Idouble}, + {"if", TOK.If}, + {"ifloat", TOK.Ifloat}, + {"import", TOK.Import}, + {"in", TOK.In}, + {"inout", TOK.Inout}, + {"int", TOK.Int}, + {"interface", TOK.Interface}, + {"invariant", TOK.Invariant}, + {"ireal", TOK.Ireal}, + {"is", TOK.Is}, + {"lazy", TOK.Lazy}, + {"long", TOK.Long}, + {"macro", TOK.Macro}, // D2.0 + {"mixin", TOK.Mixin}, + {"module", TOK.Module}, + {"new", TOK.New}, + {"null", TOK.Null}, + {"out", TOK.Out}, + {"override", TOK.Override}, + {"package", TOK.Package}, + {"pragma", TOK.Pragma}, + {"private", TOK.Private}, + {"protected", TOK.Protected}, + {"public", TOK.Public}, + {"real", TOK.Real}, + {"ref", TOK.Ref}, + {"return", TOK.Return}, + {"scope", TOK.Scope}, + {"short", TOK.Short}, + {"static", TOK.Static}, + {"struct", TOK.Struct}, + {"super", TOK.Super}, + {"switch", TOK.Switch}, + {"synchronized", TOK.Synchronized}, + {"template", TOK.Template}, + {"this", TOK.This}, + {"throw", TOK.Throw}, + {"__traits", TOK.Traits}, // D2.0 + {"true", TOK.True}, + {"try", TOK.Try}, + {"typedef", TOK.Typedef}, + {"typeid", TOK.Typeid}, + {"typeof", TOK.Typeof}, + {"ubyte", TOK.Ubyte}, + {"ucent", TOK.Ucent}, + {"uint", TOK.Uint}, + {"ulong", TOK.Ulong}, + {"union", TOK.Union}, + {"unittest", TOK.Unittest}, + {"ushort", TOK.Ushort}, + {"version", TOK.Version}, + {"void", TOK.Void}, + {"volatile", TOK.Volatile}, + {"wchar", TOK.Wchar}, + {"while", TOK.While}, + {"with", TOK.With}, // Special tokens: - {TOK.FILE, "__FILE__"}, - {TOK.LINE, "__LINE__"}, - {TOK.DATE, "__DATE__"}, - {TOK.TIME, "__TIME__"}, - {TOK.TIMESTAMP, "__TIMESTAMP__"}, - {TOK.VENDOR, "__VENDOR__"}, - {TOK.VERSION, "__VERSION__"}, - {TOK.EOF, "__EOF__"}, // D2.0 + {"__FILE__", TOK.FILE}, + {"__LINE__", TOK.LINE}, + {"__DATE__", TOK.DATE}, + {"__TIME__", TOK.TIME}, + {"__TIMESTAMP__", TOK.TIMESTAMP}, + {"__VENDOR__", TOK.VENDOR}, + {"__VERSION__", TOK.VERSION}, + {"__EOF__", TOK.EOF}, // D2.0 ]; diff -r 949a53332c66 -r 4e14cd1b24da trunk/src/dil/Lexer.d --- a/trunk/src/dil/Lexer.d Sun Dec 09 23:27:40 2007 +0100 +++ b/trunk/src/dil/Lexer.d Tue Dec 11 14:19:30 2007 +0100 @@ -10,6 +10,7 @@ import dil.Messages; import dil.HtmlEntities; import dil.CompilerInfo; +import dil.IdTable; import tango.stdc.stdlib : strtof, strtod, strtold; import tango.stdc.errno : errno, ERANGE; import tango.stdc.time : time_t, time, ctime; @@ -23,31 +24,6 @@ /// U+FFFD = �. Used to replace invalid Unicode characters. const dchar REPLACEMENT_CHAR = '\uFFFD'; -/// Global table of identifiers. Access must be synchronized. -private Identifier*[string] idTable; - -static this() -{ - foreach(ref k; keywords) - idTable[k.str] = &k; -} - -Identifier* idTableLookup(string idString) -out(id) -{ assert(id !is null); } -body -{ - synchronized - { - Identifier** id = idString in idTable; - if (id) - return *id; - auto newID = Identifier(TOK.Identifier, idString); - idTable[idString] = newID; - return newID; - } -} - /++ The Lexer analyzes the characters of a source text and produces a doubly-linked list of tokens. @@ -313,8 +289,9 @@ t.end = p; - auto id = idTableLookup(t.srcText); + auto id = IdTable.lookup(t.srcText); t.type = id.type; + t.ident = id; if (t.type == TOK.Identifier || t.isKeyword) return; @@ -326,7 +303,7 @@ assert(t.srcText == "__EOF__"); } else - assert(0, "unexpected token: " ~ t.srcText); + assert(0, "unexpected token type: " ~ Token.toString(t.type)); return; } @@ -1058,8 +1035,9 @@ t.end = p; - auto id = idTableLookup(t.srcText); + auto id = IdTable.lookup(t.srcText); t.type = id.type; + t.ident = id; if (t.type == TOK.Identifier || t.isKeyword) return; @@ -1071,7 +1049,7 @@ assert(t.srcText == "__EOF__"); } else - assert(0, "unexpected token: " ~ t.srcText); + assert(0, "unexpected token type: " ~ Token.toString(t.type)); return; } diff -r 949a53332c66 -r 4e14cd1b24da trunk/src/dil/Parser.d --- a/trunk/src/dil/Parser.d Sun Dec 09 23:27:40 2007 +0100 +++ b/trunk/src/dil/Parser.d Tue Dec 11 14:19:30 2007 +0100 @@ -14,10 +14,10 @@ import dil.Types; import dil.Enums; import dil.CompilerInfo; +import dil.IdentsEnum; +import dil.IdTable; import common; -private alias TOK T; - /++ The Parser produces an abstract syntax tree (AST) by analyzing the tokens of the provided source code. @@ -37,6 +37,8 @@ StorageClass storageClass; uint alignSize = DEFAULT_ALIGN_SIZE; + private alias TOK T; + this(char[] srcText, string filePath, InformationManager infoMan = null) { lx = new Lexer(srcText, filePath); @@ -682,11 +684,13 @@ return linkageType; } - auto ident = requireId(); - - switch (ident ? ident.identifier : null) + auto identTok = requireId(); + + ID identID = identTok ? identTok.ident.identID : ID.Null; + + switch (identID) { - case "C": + case ID.C: if (token.type == T.PlusPlus) { nT(); @@ -695,16 +699,16 @@ } linkageType = LinkageType.C; break; - case "D": + case ID.D: linkageType = LinkageType.D; break; - case "Windows": + case ID.Windows: linkageType = LinkageType.Windows; break; - case "Pascal": + case ID.Pascal: linkageType = LinkageType.Pascal; break; - case "System": + case ID.System: linkageType = LinkageType.System; break; default: @@ -2448,14 +2452,13 @@ Token* condition = requireId(); if (condition) - switch (condition.identifier) + switch (condition.ident.identID) { - case "exit": - case "success": - case "failure": + case ID.exit, ID.success, ID.failure: break; default: - // TODO: issue error msg. + // TODO: create MID.InvalidScopeIdentifier + error(condition, "'exit', 'success', 'failure' are valid scope identifiers, but not '{}';", condition.srcText); } require(T.RParen); Statement scopeBody; @@ -2724,7 +2727,7 @@ nT(); number = null; // TODO: report error: number expected after asm align statement. - error(MID.ExpectedButFound, "integer", token.srcText); + error(token, "expected an integer after align, not '{}'", token.srcText); } require(T.Semicolon); s = new AsmAlignStatement(number); @@ -2954,23 +2957,23 @@ T.Float, T.Double, T.Real: goto LAsmTypePrefix; case T.Identifier: - switch (token.identifier) + switch (token.ident.identID) { - case "near", "far", /*"byte", "short", "int",*/ - "word", "dword", "qword"/*, "float", "double", "real"*/: + case ID.near, ID.far,/* "byte", "short", "int",*/ + ID.word, ID.dword, ID.qword/*, "float", "double", "real"*/: LAsmTypePrefix: nT(); - if (token.type == T.Identifier && token.identifier == "ptr") + if (token.type == T.Identifier && token.ident is Ident.ptr) nT(); else error(MID.ExpectedButFound, "ptr", token.srcText); e = new AsmTypeExpression(parseAsmExpression()); break; - case "offset": + case ID.offset: nT(); e = new AsmOffsetExpression(parseAsmExpression()); break; - case "seg": + case ID.seg: nT(); e = new AsmSegExpression(parseAsmExpression()); break; @@ -3026,70 +3029,61 @@ e = new AsmBracketExpression(e); break; case T.Identifier: - switch (token.identifier) + auto register = token.ident; + switch (register.identID) { // __LOCAL_SIZE - case "__LOCAL_SIZE": + case ID.__LOCAL_SIZE: + nT(); e = new AsmLocalSizeExpression(); - nT(); break; // Register - case "ST": - auto register = token; + case ID.ST: nT(); // (1) - (7) - Token* number; + int number = -1; if (token.type == T.LParen) { nT(); if (token.type == T.Int32) - { - number = token; - nT(); - } + (number = token.int_), nT(); else expected(T.Int32); require(T.RParen); } e = new AsmRegisterExpression(register, number); break; - case "FS": - auto register = token; + case ID.FS: nT(); // TODO: is the colon-number part optional? - Token* number; + int number = -1; if (token.type == T.Colon) { // :0, :4, :8 nT(); - switch (token.srcText) - { - case "0", "4", "8": - number = token; - nT(); - break; - default: + if (token.type == T.Int32) + (number = token.int_), nT(); + if (number != 0 && number != 4 && number != 8) error(MID.ExpectedButFound, "0, 4 or 8", token.srcText); - } } e = new AsmRegisterExpression(register, number); break; - case "AL", "AH", "AX", "EAX", - "BL", "BH", "BX", "EBX", - "CL", "CH", "CX", "ECX", - "DL", "DH", "DX", "EDX", - "BP", "EBP", - "SP", "ESP", - "DI", "EDI", - "SI", "ESI", - "ES", "CS", "SS", "DS", "GS", - "CR0", "CR2", "CR3", "CR4", - "DR0", "DR1", "DR2", "DR3", "DR6", "DR7", - "TR3", "TR4", "TR5", "TR6", "TR7", - "MM0", "MM1", "MM2", "MM3", "MM4", "MM5", "MM6", "MM7", - "XMM0", "XMM1", "XMM2", "XMM3", "XMM4", "XMM5", "XMM6", "XMM7": - e = new AsmRegisterExpression(token); - nT(); + case ID.AL, ID.AH, ID.AX, ID.EAX, + ID.BL, ID.BH, ID.BX, ID.EBX, + ID.CL, ID.CH, ID.CX, ID.ECX, + ID.DL, ID.DH, ID.DX, ID.EDX, + ID.BP, ID.EBP, ID.SP, ID.ESP, + ID.DI, ID.EDI, ID.SI, ID.ESI, + ID.ES, ID.CS, ID.SS, ID.DS, ID.GS, + ID.CR0, ID.CR2, ID.CR3, ID.CR4, + ID.DR0, ID.DR1, ID.DR2, ID.DR3, ID.DR6, ID.DR7, + ID.TR3, ID.TR4, ID.TR5, ID.TR6, ID.TR7, + ID.MM0, ID.MM1, ID.MM2, ID.MM3, + ID.MM4, ID.MM5, ID.MM6, ID.MM7, + ID.XMM0, ID.XMM1, ID.XMM2, ID.XMM3, + ID.XMM4, ID.XMM5, ID.XMM6, ID.XMM7: + nT(); + e = new AsmRegisterExpression(register); break; default: // DotIdentifier @@ -3115,7 +3109,7 @@ { // Insert a dummy token and don't consume current one. begin = lx.insertEmptyTokenBefore(token); - prevToken = begin; + this.prevToken = begin; } } set(e, begin); @@ -3797,7 +3791,7 @@ { // Insert a dummy token and don't consume current one. begin = lx.insertEmptyTokenBefore(token); - prevToken = begin; + this.prevToken = begin; } } set(e, begin); diff -r 949a53332c66 -r 4e14cd1b24da trunk/src/dil/Token.d --- a/trunk/src/dil/Token.d Sun Dec 09 23:27:40 2007 +0100 +++ b/trunk/src/dil/Token.d Tue Dec 11 14:19:30 2007 +0100 @@ -4,11 +4,12 @@ +/ module dil.Token; import dil.Location; +import dil.Identifier; import tango.stdc.stdlib : malloc, free; import tango.core.Exception; import common; -public import dil.TokenIDs; +public import dil.TokensEnum; /++ A Token is a sequence of characters formed by the lexical analyzer. @@ -48,6 +49,7 @@ /// doubly linked list. The last token is always '}' or /// EOF in case end of source text is "q{" EOF. } + Identifier* ident; dchar dchar_; long long_; ulong ulong_; diff -r 949a53332c66 -r 4e14cd1b24da trunk/src/dil/TokensEnum.d --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/trunk/src/dil/TokensEnum.d Tue Dec 11 14:19:30 2007 +0100 @@ -0,0 +1,203 @@ +/++ + Author: Aziz Köksal + License: GPL3 ++/ +module dil.TokensEnum; +import common; + +enum TOK : ushort +{ + Invalid, + + /// Flag for whitespace tokens that must be ignored in the parsing phase. + Whitespace = 0x8000, + Illegal = 1 | Whitespace, + Comment = 2 | Whitespace, + Shebang = 3 | Whitespace, + HashLine = 4 | Whitespace, + Filespec = 5 | Whitespace, + Newline = 6 | Whitespace, + Empty = 7, + + Identifier = 8, + String, + CharLiteral, WCharLiteral, DCharLiteral, + + // Special tokens + FILE, + LINE, + DATE, + TIME, + TIMESTAMP, + VENDOR, + VERSION, + + // Number literals + Int32, Int64, Uint32, Uint64, + // Floating point number scanner relies on this order. (FloatXY + 3 == ImaginaryXY) + Float32, Float64, Float80, + Imaginary32, Imaginary64, Imaginary80, + + + // Brackets + LParen, + RParen, + LBracket, + RBracket, + LBrace, + RBrace, + + Dot, Slice, Ellipses, + + // Floating point number operators + Unordered, + UorE, + UorG, + UorGorE, + UorL, + UorLorE, + LorEorG, + LorG, + + // Normal operators + Assign, Equal, NotEqual, Not, + LessEqual, Less, + GreaterEqual, Greater, + LShiftAssign, LShift, + RShiftAssign,RShift, + URShiftAssign, URShift, + OrAssign, OrLogical, OrBinary, + AndAssign, AndLogical, AndBinary, + PlusAssign, PlusPlus, Plus, + MinusAssign, MinusMinus, Minus, + DivAssign, Div, + MulAssign, Mul, + ModAssign, Mod, + XorAssign, Xor, + CatAssign, Catenate, + Tilde, + Identity, NotIdentity, + + Colon, + Semicolon, + Question, + Comma, + Dollar, + + /* Keywords: + NB.: Token.isKeyword() depends on this list being contiguous. + */ + Abstract,Alias,Align,Asm,Assert,Auto,Body, + Bool,Break,Byte,Case,Cast,Catch,Cdouble, + Cent,Cfloat,Char,Class,Const,Continue,Creal, + Dchar,Debug,Default,Delegate,Delete,Deprecated,Do, + Double,Else,Enum,Export,Extern,False,Final, + Finally,Float,For,Foreach,Foreach_reverse,Function,Goto, + Idouble,If,Ifloat,Import,In,Inout,Int, + Interface,Invariant,Ireal,Is,Lazy,Long,Macro/+D2.0+/, + Mixin,Module,New,Null,Out,Override,Package, + Pragma,Private,Protected,Public,Real,Ref/+D2.0+/,Return, + Scope,Short,Static,Struct,Super,Switch,Synchronized, + Template,This,Throw,Traits/+D2.0+/,True,Try,Typedef,Typeid, + Typeof,Ubyte,Ucent,Uint,Ulong,Union,Unittest, + Ushort,Version,Void,Volatile,Wchar,While,With, + + HEAD, // start of linked list + EOF, + MAX +} + +alias TOK.Abstract KeywordsBegin; +alias TOK.With KeywordsEnd; +alias TOK.FILE SpecialTokensBegin; +alias TOK.VERSION SpecialTokensEnd; + +/// A table mapping each TOK to a string. +const string[] tokToString = [ + "Invalid", + + "Illegal", + "Comment", + "#! /shebang/", + "#line", + `"filespec"`, + "Newline", + "Empty", + + "Identifier", + "String", + "CharLiteral", "WCharLiteral", "DCharLiteral", + + "__FILE__", + "__LINE__", + "__DATE__", + "__TIME__", + "__TIMESTAMP__", + "__VENDOR__", + "__VERSION__", + + "Int32", "Int64", "Uint32", "Uint64", + "Float32", "Float64", "Float80", + "Imaginary32", "Imaginary64", "Imaginary80", + + "(", + ")", + "[", + "]", + "{", + "}", + + ".", "..", "...", + + "!<>=", // Unordered + "!<>", // UorE + "!<=", // UorG + "!<", // UorGorE + "!>=", // UorL + "!>", // UorLorE + "<>=", // LorEorG + "<>", // LorG + + "=", "==", "!=", "!", + "<=", "<", + ">=", ">", + "<<=", "<<", + ">>=",">>", + ">>>=", ">>>", + "|=", "||", "|", + "&=", "&&", "&", + "+=", "++", "+", + "-=", "--", "-", + "/=", "/", + "*=", "*", + "%=", "%", + "^=", "^", + "~=", "~", + "~", + "is", "!is", + + ":", + ";", + "?", + ",", + "$", + + "abstract","alias","align","asm","assert","auto","body", + "bool","break","byte","case","cast","catch","cdouble", + "cent","cfloat","char","class","const","continue","creal", + "dchar","debug","default","delegate","delete","deprecated","do", + "double","else","enum","export","extern","false","final", + "finally","float","for","foreach","foreach_reverse","function","goto", + "idouble","if","ifloat","import","in","inout","int", + "interface","invariant","ireal","is","lazy","long","macro", + "mixin","module","new","null","out","override","package", + "pragma","private","protected","public","real","ref","return", + "scope","short","static","struct","super","switch","synchronized", + "template","this","throw","__traits","true","try","typedef","typeid", + "typeof","ubyte","ucent","uint","ulong","union","unittest", + "ushort","version","void","volatile","wchar","while","with", + + "HEAD", + "EOF" +]; +static assert(tokToString.length == TOK.EOF+1);