Mercurial > projects > dang
changeset 88:eb5b2c719a39 new_gen
Major change to locations, tokens and expressions.
A location (now SourceLocation or SLoc) is only 32 bit in size -
disadvantage is that it can't find its own text. You have to go through the
new SourceManager to do that.
This has caused changes to a lot of stuff and removal of DataSource and the
old Location
Additionally Exp has gotten some location stuff, so we can give proper
error messages. Not in Decl and Stmt yet, but thats coming too.
| author | Anders Halager <halager@gmail.com> |
|---|---|
| date | Sun, 04 May 2008 18:13:46 +0200 |
| parents | 9a35a973175a |
| children | a49bb982a7b0 |
| files | ast/Exp.d ast/Stmt.d basic/SourceLocation.d basic/SourceManager.d dang/compiler.d gen/CodeGen.d lexer/Lexer.d lexer/Token.d misc/DataSource.d misc/Error.d misc/Location.d parser/Action.d parser/Parser.d sema/AstAction.d sema/DType.d sema/Declarations.d sema/ImplicitCast.d tools/AstPrinter.d tools/DotPrinter.d |
| diffstat | 19 files changed, 701 insertions(+), 257 deletions(-) [+] |
line wrap: on
line diff
--- a/ast/Exp.d Sun May 04 12:58:02 2008 +0200 +++ b/ast/Exp.d Sun May 04 18:13:46 2008 +0200 @@ -29,17 +29,36 @@ class Exp { - this(ExpType expType) + this(ExpType expType, SLoc loc) { this.expType = expType; + this.loc = loc; } + /// Get the type of the expression DType type() { return null; } + /// Indicates which type the expression is - to avoid a lot of casts ExpType expType; + + /// The environment of the expression Scope env; + int stmtIndex; + /** + The "main" location of the expression. + What exactly this represents varies but for most things its the start + while for a binary expression its the operator. + **/ + SourceLocation loc; + + /// Return the starting location of this expression + SourceLocation startLoc() { return loc; } + + /// Get the full extents of the expression + SourceRange sourceRange() { return SourceRange(loc, loc + 1); } + Exp simplify() { return this; @@ -50,7 +69,7 @@ { this(Exp exp, Exp[] args) { - super(ExpType.CallExp); + super(ExpType.CallExp, exp.loc); this.exp = exp; this.args = args; } @@ -66,6 +85,13 @@ Exp[] args; bool sret = false; + override SourceRange sourceRange() + { + SourceRange res = exp.sourceRange; + if (args.length > 0) + res = res + args[$ - 1].sourceRange; + return res; + } Exp simplify() { @@ -113,12 +139,13 @@ class AssignExp : Exp { - this(Exp identifier, Exp exp) + this(SLoc op, Exp identifier, Exp exp) { - super(ExpType.AssignExp); + super(ExpType.AssignExp, op); this.identifier = identifier; this.exp = exp; } + Exp simplify() { identifier = identifier.simplify; @@ -127,6 +154,11 @@ return this; } + override SourceRange sourceRange() + { + return identifier.sourceRange + exp.sourceRange; + } + override DType type() { return identifier.type(); } Exp identifier; @@ -150,9 +182,9 @@ char[][] getOp = ["=","==","!=","<","<=",">",">=","+","-","*","/","%"]; - this(Operator op, Exp left, Exp right) + this(SLoc op_loc, Operator op, Exp left, Exp right) { - super(ExpType.Binary); + super(ExpType.Binary, op_loc); this.op = op; this.left = left; this.right = right; @@ -187,12 +219,20 @@ return myType; } + override SLoc startLoc() { return left.startLoc(); } + + override SourceRange sourceRange() + { + return left.sourceRange + right.sourceRange; + } + char[] resultType() { if (op >= Operator.Eq && op <= Operator.Ge) return "bool"; return null; } + Exp simplify() { left = left.simplify; @@ -207,9 +247,9 @@ class NegateExp : Exp { - this(Exp exp) + this(SLoc op, Exp exp) { - super(ExpType.Negate); + super(ExpType.Negate, op); this.exp = exp; } @@ -221,14 +261,19 @@ override DType type() { return exp.type(); } + override SourceRange sourceRange() + { + return SourceRange(loc) + exp.sourceRange; + } + public Exp exp; } class DerefExp : Exp { - this(Exp exp) + this(SLoc op, Exp exp) { - super(ExpType.Deref); + super(ExpType.Deref, op); this.exp = exp; } @@ -243,22 +288,28 @@ return exp.type().asPointer.pointerOf; } + override SourceRange sourceRange() + { + return SourceRange(loc) + exp.sourceRange; + } + public Exp exp; } class IntegerLit : Exp { - this(Token t) + this(SLoc loc, char[] t) { - super(ExpType.IntegerLit); - this.token = t; - this.name = substitute(t.get,"_",""); + super(ExpType.IntegerLit, loc); + range = SourceRange(loc, loc + t.length); + this.name = substitute(t, "_", ""); } char[] get() { return name; } + Exp simplify() { return this; @@ -266,20 +317,24 @@ override DType type() { return DType.Int; } - + override SourceRange sourceRange() + { + return range; + } - Token token; char[] name; + private SourceRange range; } class MemberReference : Exp { - this(Exp target, Identifier child) + this(SLoc dot, Exp target, Identifier child) { - super(ExpType.MemberReference); + super(ExpType.MemberReference, dot); this.target = target; this.child = child; } + Exp simplify() { target = target.simplify; @@ -293,7 +348,7 @@ DStruct st = cast(DStruct)target.type; assert(st, "Only structs have members"); - if (auto t = st.typeOf(child.token.get)) + if (auto t = st.typeOf(child.name)) myType = t; // no error reporting here else assert(0, "Referencing non-existant member"); @@ -301,6 +356,13 @@ return myType; } + override SLoc startLoc() { return target.startLoc(); } + + override SourceRange sourceRange() + { + return target.sourceRange + child.sourceRange; + } + Identifier child; Exp target; private DType myType; @@ -308,11 +370,13 @@ class IndexExp : Exp { - this(Exp target, Exp index) + this(Exp target, SLoc left_bracket, Exp index, SLoc right_bracket) { - super(ExpType.Index); + super(ExpType.Index, target.startLoc); this.target = target; + this.left_bracket = left_bracket; this.index = index; + this.right_bracket = right_bracket; } override DType type() @@ -325,6 +389,11 @@ else assert(0, "Can only index pointers and arrays"); } + override SourceRange sourceRange() + { + return target.sourceRange + SourceRange(right_bracket); + } + Exp simplify() { target = target.simplify; @@ -334,13 +403,14 @@ Exp target; Exp index; + SLoc left_bracket, right_bracket; } class CastExp : Exp { - this(Identifier castType, Exp exp) + this(SLoc loc, Identifier castType, Exp exp) { - super(ExpType.CastExp); + super(ExpType.CastExp, loc); this.castType = castType; this.exp = exp; } @@ -357,6 +427,11 @@ return this; } + override SourceRange sourceRange() + { + return SourceRange(loc) + exp.sourceRange; + } + Identifier castType; Exp exp; } @@ -365,7 +440,7 @@ { this(Identifier pointerOf) { - super(ExpType.PointerIdentifier); + super(ExpType.PointerIdentifier, pointerOf.loc); this.pointerOf = pointerOf; this.name = pointerOf.name; } @@ -382,7 +457,7 @@ { this(Identifier arrayOf, IntegerLit size) { - super(ExpType.ArrayIdentifier); + super(ExpType.ArrayIdentifier, arrayOf.loc); this.arrayOf = arrayOf; this.size = Integer.parse(size.get); this.name = arrayOf.name; @@ -401,16 +476,15 @@ class Identifier : Exp { - this(Token t) + this(SLoc loc, char[] name) { - super(ExpType.Identifier); - this.token = t; - name = t.get; + super(ExpType.Identifier, loc); + this.name = name; } - protected this(ExpType t) + protected this(ExpType t, SLoc loc) { - super(t); + super(t, loc); } override DType type() @@ -423,7 +497,7 @@ this(char[] name) { - super(ExpType.Identifier); + super(ExpType.Identifier, SLoc.Invalid); this.name = name; } @@ -456,7 +530,6 @@ return this; } - Token token; char[] name; private DType myType; }
--- a/ast/Stmt.d Sun May 04 12:58:02 2008 +0200 +++ b/ast/Stmt.d Sun May 04 18:13:46 2008 +0200 @@ -8,6 +8,7 @@ ast.Decl; import sema.SymbolTable, + basic.SourceLocation, misc.Error; enum StmtType @@ -71,7 +72,7 @@ { auto i = new Identifier("ret.val"); i.env = f.env; - auto ass = new AssignExp(i, exp); + auto ass = new AssignExp(SLoc.Invalid, i, exp); ass.env = f.env; auto assStmt = new ExpStmt(ass); assStmt.env = f.env; @@ -179,7 +180,7 @@ { long[] new_values; foreach (lit; values) - new_values ~= Integer.parse(lit.token.get); + new_values ~= Integer.parse(lit.get); cases ~= Case(values, stmts, new_values); // Make sure there is no two cases with the same value @@ -193,7 +194,7 @@ auto e = new Error( "Can't have multiple cases with the same value." " Values appearing in multiple cases: %0"); - e.loc(values[0].token.location); + //e.loc(values[0].token.location); all_values = Array.intersectionOf(old_values, new_values); char[][] vals;
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/basic/SourceLocation.d Sun May 04 18:13:46 2008 +0200 @@ -0,0 +1,145 @@ +module basic.SourceLocation; + +/// Shorter alias for SourceLocation +public alias SourceLocation SLoc; + +/// SourceLocation points to a place in some buffer +struct SourceLocation +{ + /// Returns true, if the location is from a real file + bool isReal() { return (val & 0x80_00_00_00) == 0; } + /// Returns true, if the location is not from a real file + bool isVirtual() { return (val & 0x80_00_00_00) != 0; } + + /// Check if this location is invalid or not + bool isValid() { return val != uint.max; } + /// ditto + bool isInvalid() { return val == uint.max; } + + /** + Extracts the file id. + + Warning: In release mode this may return junk, if the loc is not from a + file + **/ + uint fileID() { + assert(isValid, "Location is invalid"); + assert(isReal, "You can only extract fileID from a real location"); + // Here we can rely on two facts, first that the high bit is zero + // since its a real position, second that FileOffset is saved in the + // high end, so all we need is some shifting + return val >> Bits.FileOffset; + } + + /** + Extracts the offset into the "file". (actually in to the referenced + chunk) + + Warning: In release mode this may return junk, if the loc is not from a + file + **/ + uint fileOffset() { + assert(isValid, "Location is invalid"); + assert(isReal, "You can only extract fileOffset from real locations"); + // FileOffset is stored in the lower bits, so all that is needed is a + // binary and with all ones in the lower bits. + return val & (1 << Bits.FileOffset) - 1; + } + + /// Get a new location, placed n bytes after the given location + SourceLocation opAdd(int n) + { + SourceLocation res = *this; + res.val += n; + return res; + } + + /// Get a new location, placed n bytes before the given location + SourceLocation opSub(int n) + { + SourceLocation res = *this; + res.val -= n; + return res; + } + + /// Creates a SourceLocation from a File ID + static SourceLocation fromFileID(uint fileID) + { + assert(fileID < Bits.MaxFileID, "To large fileID"); + SourceLocation res; + res.val = fileID << Bits.FileID; + return res; + } + + /** + Used for invalid/unknown locations. (also the default value, but this is + more explicit) + **/ + static const SourceLocation Invalid = {val: uint.max}; + +private: + /** + A SourceLocation consists of 1 bit, indicating real or virtual, meaning + if the location points to a file(real), a string mixin or has been + affected by #line(virtual). That information is saved in the most + significant bit. + The rest depends on which type we are dealing with. + Real: + 13 bits for a file identifier + 18 bits for offset into that "file" (one file may be split) + Virtual: + Unknown for now. Likely skewed toward more ids and some meta data + An invalid location is uint.max, this might happen by accident but its + unlikely. + **/ + uint val = uint.max; + + /** + This enum contains some constants that are useful for manipulating + SourceLocation's, like the size of various “members” of val. + **/ + static enum Bits { + /// Number of bits used for the offset into file buffers + FileOffset = 18, + /// Number of bits used to identify which file buffer this is from + FileID = 31 - FileOffset, + + /// Indicates how much can be indexed within one block(2^FileOffset) + MaxFileOffset = 1 << FileOffset, + MaxFileID = 1 << FileID, + } +} + +/// A simple pair used to describe a range in a buffer and not just a point. +struct SourceRange +{ + SourceLocation begin, end; + + static SourceRange opCall(SourceLocation loc) + { + return SourceRange(loc, loc + 1); + } + + static SourceRange opCall(SourceLocation begin, SourceLocation end) + { + SourceRange res; + res.begin = begin; + res.end = end; + return res; + } + + bool isValid() { return begin.isValid && end.isValid; } + bool isInvalid() { return begin.isInvalid || end.isInvalid; } + + bool isReal() { return begin.isReal && end.isReal; } + + /// Get a new range spanning both ranges + SourceRange opAdd(SourceRange that) + { + assert(this.isValid && that.isValid, "Invalid range"); + return SourceRange( + this.begin.val < that.begin.val? this.begin : that.begin, + this.end.val > that.end.val? this.end : that.end); + } +} +
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/basic/SourceManager.d Sun May 04 18:13:46 2008 +0200 @@ -0,0 +1,248 @@ +module basic.SourceManager; + +import tango.core.Memory : GC; +import tango.io.UnicodeFile; +import tango.io.Stdout; +import tango.text.convert.Layout; + +public import basic.SourceLocation; + +private alias char[] string; + +/** + SourceManager is used to handle input files, by loading them in in chunks + that can be referenced elsewhere. + + It will also help extract the line/col of locations and convert between + real and virtual locations + **/ +class SourceManager +{ + this() + { + layout = new Layout!(char); + } + + /** + Will load in the file belonging to the filename + + filename = The file to load. Theres some assumptions about this file. + 1. The file has a BOM or is valid utf-8 + 2. The file is not empty, unreadable, a folder etc. + **/ + SourceLocation addFile(string filename) + { + scope file = new UnicodeFile!(char)(filename, Encoding.UTF_8); + auto file_data = file.read(); + return createCheckpoints(file_data, filename); + } + + /** + Returns a string slice containing the part of the file after loc (a + pointer might be better, it allows negative indexing) + **/ + string getRawData(SourceLocation loc) + { + return checkpoints[loc.fileID].data[loc.fileOffset .. $]; + } + + /** + Extracts the line number of the given location + O("file size") if cache isn't built, O(log "lines in file") else + **/ + uint getLineNumber(SourceLocation loc) + { + assert(loc.isValid, "Location is invalid"); + assert(loc.isReal, "Virtual locations not supported yet"); + assert(loc.fileID < checkpoints.length, "Non-existent location"); + + CP* cp = &checkpoints[loc.fileID]; + auto cache = &linecache[cp.meta_index]; + if (!cache.isCached) + cache.build(cp.data); + return cache.lineOf(getFileOffset(loc)); + } + + /** + Extracts the full byte offset into a file, at which a location + is pointing. + **/ + uint getFileOffset(SourceLocation loc) + { + return loc.fileOffset + + checkpoints[loc.fileID].part * loc.Bits.MaxFileOffset; + } + + /** + Extracts a string containing the entire line loc appears in. + **/ + string getLine(SourceLocation loc) + { + // The line is extracted by getting two pointers to the exact location + // and decreasing one until the nearest newline while the other ptr is + // increased to the nearest newline. + CP* cp = &checkpoints[loc.fileID]; + char* ptr = cp.data.ptr + loc.fileOffset; + char* ptr_lo = ptr; + while (ptr_lo != cp.data.ptr && *ptr_lo != '\n' && *ptr_lo != '\r') + --ptr_lo; + while (ptr != cp.data.ptr + cp.data.length && *ptr != '\n' && *ptr != '\r') + ++ptr; + return ptr_lo[0 .. ptr - ptr_lo]; + } + + /** + Get the original source text of a SourceRange + **/ + string getText(SourceRange loc) + { + assert(loc.isValid, "Range is invalid"); + assert(loc.isReal, "Virtual locations not supported yet"); + auto begin = getFileOffset(loc.begin); + auto end = getFileOffset(loc.end); + return checkpoints[loc.begin.fileID].data.ptr[begin .. end]; + } + + /** + Get the original source text + **/ + string getText(SourceLocation loc, size_t length) + { + return getText(SourceRange(loc, loc + length)); + } + + /** + Convert a location into a string. Something like "file(line)" + **/ + string getLocationAsString(SourceLocation loc) + { + assert(loc.isValid, "Location is invalid"); + return layout.convert("{}({})", + checkpoints[loc.fileID].filename, + getLineNumber(loc)); + } + string getLocationAsString(SourceRange loc) + { + return layout.convert("{}({}:{})", + checkpoints[loc.begin.fileID].filename, + getFileOffset(loc.begin), + getFileOffset(loc.end)); + } + +private: + synchronized + SourceLocation createCheckpoints(string data, string source_file) + { + // The line-cache is added, but not built, + // getLineNumber makes sure it is called when needed. + linecache ~= FileLineCache(); + uint meta_index = linecache.length - 1; + + // SourceLocation's can only index relatively short buffers, therefore + // the file is split into several checkpoints. + uint checkpoint_counter = checkpoints.length; + while (data.length > 0) + { + uint to_take = min(data.length, SourceLocation.Bits.MaxFileOffset); + checkpoints ~= + CP(source_file, + data[0 .. to_take], + checkpoint_counter++, + meta_index); + data = data[to_take .. $]; + // Stdout("Taking ")(to_take)(" from ")(source_file).newline; + } + checkpoint_counter = checkpoints.length - checkpoint_counter; + return SourceLocation.fromFileID(checkpoint_counter); + } + + /// Contains the read/generated data. + CP[] checkpoints; + /// Cache used to speed up finding of line-starts. + FileLineCache[] linecache; + /// Used for formatting locations as strings. + Layout!(char) layout; + + // These really should be magically available everywhere and templated. + int min(int a, int b) { return a < b? a : b; } + int max(int a, int b) { return a >= b? a : b; } + + // A Check Point is used to store a file in multiple parts, to overcome + // the limitation of SourceLocation only having a rather limited amount of + // bits to index any one file. + struct CP + { + // read-only + char[] filename; + // ditto + char[] data; + // ditto + uint part = 0; + // ditto + uint meta_index = 0; + } + + struct FileLineCache + { + /// Contains the offset of the i'th line on index i + uint[] line_starts; + + /// Indicates weather the cache has been built or not + bool isCached = false; + + /** + This method does a binary search to find the line that contains the + given offset. + **/ + uint lineOf(uint offset) + { + size_t beg = 0, + end = line_starts.length, + mid = end >> 1; + + while( beg < end ) + { + if( line_starts[mid] <= offset ) + beg = mid + 1; + else + end = mid; + mid = beg + ( end - beg ) / 2; + } + return mid; + } + + /** + Builds the cache data - always make sure this has been called before + calling lineOf. + **/ + void build(char[] data) + { + // j starts at 1, because we need an additional place in the array + // to indicate that line 1 starts at index 0. + size_t j = 1; + char* it = data.ptr, end = data.ptr + data.length; + for (; it != end; ++it) + if (*it == '\n') + ++j; + // Allocate without initialization. Saves a bit of time + line_starts.length = j; + line_starts[0] = 0; + + // Go over the data again, writing the line starts in our new array + j = 1; + for (size_t i = 0; i < data.length; i++) + { + if (data[i] == '\n') + line_starts[j++] = i; + else if (data[i] == '\r') + { + line_starts[j++] = i; + i += cast(size_t)(data[i+1] == '\n'); + } + } + + isCached = true; + } + } +} +
--- a/dang/compiler.d Sun May 04 12:58:02 2008 +0200 +++ b/dang/compiler.d Sun May 04 18:13:46 2008 +0200 @@ -8,7 +8,7 @@ import lexer.Lexer, parser.Parser; -import misc.DataSource; +import basic.SourceManager; import ast.Decl; @@ -76,7 +76,7 @@ Signal!(Lexer) postLex; Signal!(Lexer) preParse; - Signal!(Decl[], DataSource) postParse; + Signal!(Decl[], SourceManager) postParse; preStart.attach(&checkFiles); @@ -128,7 +128,7 @@ auto what = options["what-to-do"]; if (what == "" || what == "gen-llvm") postParse.attach( - (Decl[] decls, DataSource src) { + (Decl[] decls, SourceManager sm) { StopWatch w; w.start; auto llvmGen = new CodeGen(); llvmGen.gen(decls, optimize, inline); @@ -136,7 +136,7 @@ }); else if (what == "dot") postParse.attach( - (Decl[] decls, DataSource src) { + (Decl[] decls, SourceManager sm) { StopWatch w; w.start; auto print = new DotPrinter(); print.print(decls); @@ -144,21 +144,24 @@ }); else if (what == "code") postParse.attach( - (Decl[] decls, DataSource src) { + (Decl[] decls, SourceManager sm) { StopWatch w; w.start; - auto print = new AstPrinter(src); + auto print = new AstPrinter(sm); print.print(decls); timings ~= Measurement("Converting AST to text", w.stop); }); + SourceManager src_mgr = new SourceManager; + StopWatch total; total.start; - foreach(file ; filesToHandle) + foreach (file; filesToHandle) { preLex(file); - auto src = DataSource(file); - auto lexer = new Lexer(src); + auto start = src_mgr.addFile(file); + Stdout(file).newline; + auto lexer = new Lexer(start, src_mgr); postLex(lexer); preParse(lexer); @@ -166,7 +169,8 @@ StopWatch watch; watch.start; auto parser = new Parser; - auto decls = cast(Decl[])parser.parse(lexer, new AstAction); + auto action = new AstAction(src_mgr); + auto decls = cast(Decl[])parser.parse(src_mgr, lexer, action); timings ~= Measurement("Lex + Parse", watch.stop); StopWatch watch2; @@ -191,7 +195,7 @@ timings ~= Measurement(" - Extracting declarations", declarations); timings ~= Measurement(" - Making casts explicit", implicit_casts); - postParse(decls, src); + postParse(decls, src_mgr); } timings ~= Measurement("Total", total.stop);
--- a/gen/CodeGen.d Sun May 04 12:58:02 2008 +0200 +++ b/gen/CodeGen.d Sun May 04 18:13:46 2008 +0200 @@ -136,7 +136,7 @@ table.leaveScope; - //m.verify(); + debug m.verify(); if(optimize) m.optimize(inline);
--- a/lexer/Lexer.d Sun May 04 12:58:02 2008 +0200 +++ b/lexer/Lexer.d Sun May 04 18:13:46 2008 +0200 @@ -1,7 +1,7 @@ module lexer.Lexer; import misc.Error, - misc.DataSource; + basic.SourceManager; import lexer.Token, lexer.Keyword; @@ -20,29 +20,26 @@ /** Create a new Lexer. - - params: - source = The source to tokenize. - */ - - this (DataSource source) + this(SourceLocation start, SourceManager src_mgr) { - this.source = source; + sm = src_mgr; + start_loc = start; position = 0; + source = sm.getRawData(start_loc); charTable.length = 256; - foreach( char c ; "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ_") + foreach (c; "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ_") charTable[c] = CharType.Letter; - foreach( char c ; "0123456789") + foreach (c; "0123456789") charTable[c] = CharType.Number; - foreach( char c ; "(){}[];:.,=!<>+-*/%") + foreach (c; "(){}[];:.,=!<>+-*/%") charTable[c] = CharType.Symbol; - foreach( char c ; " \n") + foreach (c; " \n") charTable[c] = CharType.Whitespace; symbolFunctions.length = 256; @@ -75,13 +72,13 @@ return: A Token - Token.type is TokType.EOF if there is no more tokens in the file. */ - Token next () + Token next() { switch (getNextChar) { case CharType.EOF: - Location l; - return Token (Tok.EOF, l, 0); + SLoc loc; + return Token(Tok.EOF, loc, 0); case CharType.Whitespace: position += 1; @@ -105,10 +102,10 @@ return: A Token - Token.type is TokType.EOF if there is no more tokens in the file. */ - Token peek ( int skip = 0) + Token peek(int skip = 0) { int oldPosition = this.position; - while(skip-- > 0) + while (skip-- > 0) this.next; Token t = this.next; this.position = oldPosition; @@ -128,47 +125,47 @@ private: Token eq() { - if(source.data[position] == '=') - return Token(Tok.Eq, Location(position++ - 1, source), 2); - return Token(Tok.Assign, Location(position - 1, source), 1); + if(source[position] == '=') + return Token(Tok.Eq, Loc(position++ - 1), 2); + return Token(Tok.Assign, Loc(position - 1), 1); } Token openBrace() { - return Token(Tok.OpenBrace, Location(position - 1, source), 1); + return Token(Tok.OpenBrace, Loc(position - 1), 1); } Token closeBrace() { - return Token(Tok.CloseBrace, Location(position - 1, this.source), 1); + return Token(Tok.CloseBrace, Loc(position - 1), 1); } Token openParentheses() { - return Token(Tok.OpenParentheses, Location(position - 1, this.source), 1); + return Token(Tok.OpenParentheses, Loc(position - 1), 1); } Token closeParentheses() { - return Token(Tok.CloseParentheses, Location(position - 1, this.source), 1); + return Token(Tok.CloseParentheses, Loc(position - 1), 1); } Token openBracket() { - return Token(Tok.OpenBracket, Location(position - 1, this.source), 1); + return Token(Tok.OpenBracket, Loc(position - 1), 1); } Token closeBracket() { - return Token(Tok.CloseBracket, Location(position - 1, source), 1); + return Token(Tok.CloseBracket, Loc(position - 1), 1); } Token seperator() { - return Token(Tok.Seperator, Location(position - 1, source), 1); + return Token(Tok.Seperator, Loc(position - 1), 1); } Token colon() { - return Token(Tok.Colon, Location(position - 1, this.source), 1); + return Token(Tok.Colon, Loc(position - 1), 1); } Token dot() { int pos = 0; while(getNextChar(0) == CharType.Number || - this.source.data[position + pos + 1] == '_') + this.source[position + pos + 1] == '_') { if(getNextChar(0) == CharType.Number) { @@ -177,61 +174,61 @@ } pos++; } - return Token(Tok.Dot, Location(position - 1, this.source), 1); + return Token(Tok.Dot, Loc(position - 1), 1); } Token comma() { - return Token(Tok.Comma, Location(position - 1, this.source), 1); + return Token(Tok.Comma, Loc(position - 1), 1); } Token ne() { - if(source.data[position] == '=') - return Token(Tok.Ne, Location(position++ - 1, this.source), 2); - return Token(Tok.Not, Location(position - 1, this.source), 1); + if(source[position] == '=') + return Token(Tok.Ne, Loc(position++ - 1), 2); + return Token(Tok.Not, Loc(position - 1), 1); } Token le() { - if(source.data[position] == '=') - return Token(Tok.Le, Location(position++ - 1, this.source), 2); - return Token(Tok.Lt, Location(position - 1, this.source), 1); + if(source[position] == '=') + return Token(Tok.Le, Loc(position++ - 1), 2); + return Token(Tok.Lt, Loc(position - 1), 1); } Token ge() { - if(source.data[position] == '=') - return Token(Tok.Ge, Location(position++ - 1, this.source), 2); - return Token(Tok.Gt, Location(position - 1, this.source), 1); + if(source[position] == '=') + return Token(Tok.Ge, Loc(position++ - 1), 2); + return Token(Tok.Gt, Loc(position - 1), 1); } Token plus() { - return Token(Tok.Plus, Location(position - 1, this.source), 1); + return Token(Tok.Plus, Loc(position - 1), 1); } Token minus() { - return Token(Tok.Minus, Location(position - 1, this.source), 1); + return Token(Tok.Minus, Loc(position - 1), 1); } Token star() { - return Token(Tok.Star, Location(position - 1, this.source), 1); + return Token(Tok.Star, Loc(position - 1), 1); } Token slash() { - switch(source.data[position]) + switch(source[position]) { case '/': while(getNextChar != CharType.EOF) { - if(source.data[position++] == '\n') + if(source[position++] == '\n') return this.next; } - return Token(Tok.EOF, Location(position, this.source), 0); + return Token(Tok.EOF, Loc(position), 0); case '*': position += 2; while(getNextChar != CharType.EOF) { ++position; - if(source.data[position-2] == '*') - if(source.data[position-1] == '/') + if(source[position-2] == '*') + if(source[position-1] == '/') return this.next; } throw error(__LINE__, "Unexpected end of file. Unclosed comment block"); @@ -242,15 +239,15 @@ while(getNextChar != CharType.EOF) { ++position; - if(source.data[position-2] == '+') - if(source.data[position-1] == '/') + if(source[position-2] == '+') + if(source[position-1] == '/') { position++; nesting--; } - if(source.data[position-2] == '/') - if(source.data[position-1] == '+') + if(source[position-2] == '/') + if(source[position-1] == '+') { nesting++; position++; @@ -262,13 +259,13 @@ throw error(__LINE__, "Unexpected end of file. Unclosed comment block"); default: - return Token(Tok.Slash, Location(position - 1, this.source), 1); + return Token(Tok.Slash, Loc(position - 1), 1); } } Token percent() { - return Token(Tok.Percent, Location(position - 1, this.source), 1); + return Token(Tok.Percent, Loc(position - 1), 1); } Token lexNumber () @@ -287,24 +284,24 @@ case CharType.Number: break; case CharType.Symbol: - if(this.source.data[position+i] == '.') + if(this.source[position+i] == '.') { if(dot) throw error(__LINE__,"Only one '.' is allowed in an floating number") - .tok(Token(Tok.Float, Location(position + i, this.source), 1)); + .tok(Token(Tok.Float, Loc(position + i), 1)); dot = true; break; } end = true; continue; case CharType.Letter: - if(this.source.data[position+i] == '_') + if(this.source[position+i] == '_') break; - if (this.source.data[position+i] == 'e' || - this.source.data[position+i] == 'E') + if (this.source[position+i] == 'e' || + this.source[position+i] == 'E') { if (e) - throw error(__LINE__,"Only one '"~this.source.data[position+i] + throw error(__LINE__,"Only one '"~this.source[position+i] ~"' is allowed in an floating number"); e = true; break; @@ -321,12 +318,12 @@ position += i; - return Token(Tok.Integer, Location(position - i, this.source), i); + return Token(Tok.Integer, Loc(position - i), i); } Token lexSymbol () { - Token t = symbolFunctions[source.data[position++]](); + Token t = symbolFunctions[source[position++]](); return t; } @@ -344,11 +341,11 @@ } } - Token t = Token(Tok.Identifier, Location(position, source), i); + Token t = Token(Tok.Identifier, Loc(), i); if (!hasNumber) { - char[] str = source.data[position .. position + i]; + char[] str = source[position .. position + i]; if(str in keywords) t.type = keywords[str]; } @@ -360,10 +357,10 @@ CharType getNextChar(int offset = 0) { - if (position + offset >= this.source.data.length) + if (position + offset >= this.source.length) return CharType.EOF; - char current = source.data[position + offset]; + char current = source[position + offset]; CharType c = charTable[current]; @@ -376,11 +373,20 @@ Error error(uint line, char[] msg) { - return (new Error(msg)).loc(Location(position, source)); + return (new Error(msg));//.loc(Loc(position)); } + private final SourceLocation Loc(int pos = -1) + { + if (pos < 0) + return start_loc + position; + return start_loc + pos; + } + + SourceManager sm; + SourceLocation start_loc; int position; - DataSource source; + char[] source; Error[] errors; CharType[] charTable; Token delegate()[] symbolFunctions; @@ -396,3 +402,4 @@ EOF } +
--- a/lexer/Token.d Sun May 04 12:58:02 2008 +0200 +++ b/lexer/Token.d Sun May 04 18:13:46 2008 +0200 @@ -1,7 +1,7 @@ module lexer.Token; public -import misc.Location; +import basic.SourceLocation; import Integer = tango.text.convert.Integer; @@ -15,14 +15,14 @@ struct Token { Tok type; - Location location; + SLoc location; uint length; /** Create a new token with a Tok type, Location in source and a length of how many chars the Token span in the source */ - static Token opCall (Tok type, Location location, uint length) + static Token opCall (Tok type, SLoc location, uint length) { Token t; t.type = type; @@ -44,18 +44,11 @@ */ char[] toString () { - return this.getType()~": Len: "~Integer.toString(this.length) - ~", Loc: "~location.toString; + return this.getType()~": Len: "~Integer.toString(this.length); } - /** - Get the string in the source that matches what this Token is - covering. - */ - char[] get () - { - return location.get(length); - } + /// Get the range of this token + SourceRange asRange() { return SourceRange(location, location + length); } /** Returns true if the type of this token is a basic type (int, float, ...).
--- a/misc/DataSource.d Sun May 04 12:58:02 2008 +0200 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,25 +0,0 @@ -module misc.DataSource; - -import tango.io.UnicodeFile; - -struct DataSource -{ - char[] name; - char[] data; - - static DataSource opCall(char[] name) - { - DataSource source; - - auto file = new UnicodeFile!(char)(name, Encoding.UTF_8); - - source.name = name; - source.data = file.read(); - return source; - } - - char[] get(uint position, ushort len) - { - return data[position .. position+len]; - } -}
--- a/misc/Error.d Sun May 04 12:58:02 2008 +0200 +++ b/misc/Error.d Sun May 04 18:13:46 2008 +0200 @@ -23,6 +23,7 @@ { char[256] tmp = void; char[] msg = layout(tmp, args); + /* if (location.source.name.length > 0) msg = location.toString ~ ": " ~ msg; else @@ -57,7 +58,7 @@ msg ~= "\n "; msg ~= marks; } - +*/ return msg; } @@ -104,14 +105,17 @@ return arg(sym.type.name ~ " " ~ sym.id.get); } + /* Error loc(Location loc) { location = loc; return this; } + */ Error tok(Token tok) { + /* if (toks.length > 0) assert(tok.location.source == toks[0].location.source, "Tokens must come from the same source"); @@ -121,12 +125,13 @@ loc = tok.location; } toks ~= tok; + */ return this; } private: char[][] args; - Location location; + //Location location; Token[] toks; Token main_tok; }
--- a/misc/Location.d Sun May 04 12:58:02 2008 +0200 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,31 +0,0 @@ -module misc.Location; - -import misc.DataSource; - -import tango.text.convert.Integer, - tango.text.Util; - -struct Location -{ - uint position; - DataSource source; - - char[] toString () - { - int lineNumber = 0; - char[] end_line; - foreach (line; lines(source.get(0, position))) - { - ++lineNumber; - end_line = line; - } - return source.name - ~ "(" ~ .toString(lineNumber) - ~ ":" ~ .toString(end_line.length) ~ ")"; - } - - char[] get(uint length) - { - return source.get(position, length); - } -}
--- a/parser/Action.d Sun May 04 12:58:02 2008 +0200 +++ b/parser/Action.d Sun May 04 18:13:46 2008 +0200 @@ -31,11 +31,6 @@ return id; } Token tok; - - char[] toString() - { - return tok.get; - } } class PointerId : Id @@ -248,7 +243,7 @@ /** Binary operator. */ - ExprT actOnBinaryOp(Operator op, ExprT l, ExprT r) + ExprT actOnBinaryOp(SLoc op_loc, Operator op, ExprT l, ExprT r) { return null; } @@ -258,7 +253,7 @@ The left hand side can be any expression, but its only possible to look up an identifier. */ - ExprT actOnMemberReference(ExprT lhs, Location op, Id member) + ExprT actOnMemberReference(ExprT lhs, SourceLocation op, Id member) { return null; } @@ -287,7 +282,7 @@ /** Cast expression. */ - ExprT actOnCastExpr(Id type, ExprT exp) + ExprT actOnCastExpr(ref Token _cast, Id type, ExprT exp) { return null; }
--- a/parser/Parser.d Sun May 04 12:58:02 2008 +0200 +++ b/parser/Parser.d Sun May 04 18:13:46 2008 +0200 @@ -7,7 +7,8 @@ import misc.Error; -import basic.SmallArray; +import basic.SmallArray, + basic.SourceManager; import tango.io.Stdout, Integer = tango.text.convert.Integer; @@ -20,10 +21,11 @@ alias Object Decl; public: - Decl[] parse(Lexer lexer, Action act) + Decl[] parse(SourceManager sm, Lexer lexer, Action act) { + this.sm = sm; this.lexer = lexer; - action = act; + this.action = act; Decl[] declarations; @@ -427,7 +429,7 @@ lexer.next(); int q = op.leftAssoc? 1 + op.prec : op.prec; auto exp2 = parseExpression(q); - exp = action.actOnBinaryOp(op.operator, exp, exp2); + exp = action.actOnBinaryOp(next.location, op.operator, exp, exp2); next = lexer.peek(); } @@ -469,24 +471,28 @@ } } else if (next.type == Tok.Cast) - return parseCast(); + return parseCast(next); else if (next.type == Tok.Integer) return action.actOnNumericConstant(next); - throw error(__LINE__, "Expected expression, not '"~next.getType~"'").tok(next); + throw error(__LINE__, "Expected expression, not '%0'") + .tok(next) + .arg(next.getType); assert(0, "Should not happen"); } - Exp parseCast() + Exp parseCast(ref Token _cast) { require(Tok.OpenParentheses); auto next = lexer.next; if(!next.isBasicType && !next.isIdentifier) - throw error(__LINE__, "Expected cast type, not "~next.get).tok(next); + throw error(__LINE__, "Expected cast type, not %0") + .tok(next) + .arg(next.getType); require(Tok.CloseParentheses); auto exp = P(); - return action.actOnCastExpr(Id(next), exp); + return action.actOnCastExpr(_cast, Id(next), exp); } struct UnOp @@ -565,10 +571,10 @@ Error error(uint line, char[] errMsg) { - Location loc = lexer.peek.location; + SLoc loc = lexer.peek.location; auto e = new Error("Parser.d(" ~ Integer.toString(line) ~ "): " ~errMsg); - e.loc(loc); + //e.loc(loc); return e; } @@ -584,4 +590,5 @@ } Lexer lexer; + SourceManager sm; }
--- a/sema/AstAction.d Sun May 04 12:58:02 2008 +0200 +++ b/sema/AstAction.d Sun May 04 18:13:46 2008 +0200 @@ -4,7 +4,8 @@ import lexer.Token; -import misc.Error; +import misc.Error, + basic.SourceManager; import ast.Exp, ast.Stmt, @@ -19,14 +20,25 @@ */ class AstAction : Action { - Identifier handleType(Id type) + this(SourceManager sm) + { + this.sm = sm; + } + private SourceManager sm; + + private Identifier handleType(Id type) { if(auto t = cast(PointerId)type) return new PointerIdentifier(handleType(t.id)); if(auto t = cast(ArrayId)type) return new ArrayIdentifier(handleType(t.id), cast(IntegerLit)t.number); else - return new Identifier(type.tok); + return identifierFromTok(type.tok); + } + + private Identifier identifierFromTok(Token t) + { + return new Identifier(t.location, sm.getText(t.asRange)); } @@ -35,9 +47,9 @@ { Exp exp = cast(Exp)init; if(type.tok.type == Tok.Struct) - return new StructDecl(new Identifier(id.tok)); + return new StructDecl(identifierFromTok(id.tok)); else - return new VarDecl(handleType(type), new Identifier(id.tok), exp); + return new VarDecl(handleType(type), identifierFromTok(id.tok), exp); } override void actOnStructMember(DeclT decl, ref Id type, ref Id name, ExprT init) @@ -45,28 +57,30 @@ Exp exp = cast(Exp)init; StructDecl st = cast(StructDecl)decl; st.addMember( - new Identifier(type.tok), - new Identifier(name.tok), + identifierFromTok(type.tok), + identifierFromTok(name.tok), exp); } - override ExprT actOnMemberReference(ExprT lhs, Location op, Id member) + override ExprT actOnMemberReference(ExprT lhs, SLoc op, Id member) { - return new MemberReference(cast(Exp)lhs, new Identifier(member.tok)); + Exp exp = cast(Exp)lhs; + Identifier id = identifierFromTok(member.tok); + return new MemberReference(op, exp, id); } override DeclT actOnStartOfFunctionDef(ref Id type, ref Id name) { - return new FuncDecl(new Identifier(type.tok), new Identifier(name.tok)); + return new FuncDecl(identifierFromTok(type.tok), identifierFromTok(name.tok)); } override void addFuncArg(DeclT func, Id type, Id name) { FuncDecl fd = cast(FuncDecl)func; if(name) - fd.addParam(handleType(type), new Identifier(name.tok)); + fd.addParam(handleType(type), identifierFromTok(name.tok)); else - fd.addParam(new Identifier(type.tok)); + fd.addParam(identifierFromTok(type.tok)); } override DeclT actOnEndOfFunction(DeclT func, StmtT stmts) @@ -121,31 +135,34 @@ // -- Expressions -- override ExprT actOnNumericConstant(Token c) { - return new IntegerLit(c); + return new IntegerLit(c.location, sm.getText(c.asRange)); } override ExprT actOnIdentifierExp(Id id) { - return new Identifier(id.tok); + return identifierFromTok(id.tok); } - override ExprT actOnBinaryOp(Operator op, ExprT l, ExprT r) + override ExprT actOnBinaryOp(SLoc op_loc, Operator op, ExprT l, ExprT r) { Exp left = cast(Exp)l; Exp right = cast(Exp)r; if (op == Operator.Assign) - return new AssignExp(left, right); + return new AssignExp(op_loc, left, right); else - return new BinaryExp(cast(BinaryExp.Operator)op, left, right); + { + BinaryExp.Operator bin_op = cast(BinaryExp.Operator)op; + return new BinaryExp(op_loc, bin_op, left, right); + } } override ExprT actOnUnaryOp(Token op, ExprT operand) { Exp target = cast(Exp)operand; if (op.type == Tok.Minus) - return new NegateExp(target); + return new NegateExp(op.location, target); if (op.type == Tok.Star) - return new DerefExp(target); + return new DerefExp(op.location, target); assert(0, "Only valid unary expressions are -x and *x"); } @@ -156,14 +173,19 @@ return new CallExp(f, arguments); } - override ExprT actOnCastExpr(Id id, ExprT exp) + override ExprT actOnCastExpr(ref Token _cast, Id id, ExprT exp) { - return new CastExp(new Identifier(id.tok), cast(Exp)exp ); + Exp target = cast(Exp)exp; + Identifier target_type = identifierFromTok(id.tok); + return new CastExp(_cast.location, target_type, target); } - override ExprT actOnIndexEpr(ExprT arr, ref Token, ExprT index, ref Token) + override ExprT + actOnIndexEpr(ExprT arr, ref Token lb, ExprT index, ref Token rb) { - return new IndexExp(cast(Exp)arr, cast(Exp)index); + Exp target = cast(Exp)arr; + Exp idx = cast(Exp)index; + return new IndexExp(target, lb.location, idx, rb.location); } }
--- a/sema/DType.d Sun May 04 12:58:02 2008 +0200 +++ b/sema/DType.d Sun May 04 18:13:46 2008 +0200 @@ -8,14 +8,13 @@ class DType { private char[] id; - private Location loc; + private SourceLocation loc; public DType actual; this(Identifier id, DType actual = null) { - Token temp = id.token; - this.id = temp.get; - this.loc = temp.location; + this.id = id.name; + this.loc = id.startLoc(); this.actual = actual is null? this : actual; } @@ -77,7 +76,7 @@ } char[] name() { return id; } - Location getLoc() { return loc; } + SourceLocation getLoc() { return loc; } int byteSize() { return 0; } /**
--- a/sema/Declarations.d Sun May 04 12:58:02 2008 +0200 +++ b/sema/Declarations.d Sun May 04 18:13:46 2008 +0200 @@ -22,16 +22,16 @@ if(symbol is null) throw error(__LINE__, "Undefined identifier: '%0'") - .arg(i.get) - .loc(i.token.location); + .arg(i.get); + //.loc(i.token.location); } override void visitVarDecl(VarDecl d) { if(!d.env.findType(d.varType)) throw error(__LINE__, "Undefined type: '%0'") - .arg(d.varType.get) - .loc(d.varType.token.location); + .arg(d.varType.get); + //.loc(d.varType.token.location); visitExp(d.identifier); if (d.init) @@ -63,8 +63,8 @@ throw error(__LINE__, "%0 %1 has no member %2") .arg(st.name) .arg(target.get) - .arg(child.get) - .tok(child.token); + .arg(child.get); + //.tok(child.token); break; case ExpType.MemberReference: break;
--- a/sema/ImplicitCast.d Sun May 04 12:58:02 2008 +0200 +++ b/sema/ImplicitCast.d Sun May 04 18:13:46 2008 +0200 @@ -5,7 +5,8 @@ import tango.io.Stdout; -import misc.Error; +import misc.Error, + basic.SourceLocation; class ImplicitCast : Visitor!(void) { @@ -24,6 +25,7 @@ throw error(__LINE__, "Cannot make implicit cast"); auto castExp = new CastExp( + SLoc.Invalid, new Identifier(exp.left.type.name), exp.right); castExp.env = exp.env; @@ -36,6 +38,7 @@ throw error(__LINE__, "Cannot make implicit cast"); auto castExp = new CastExp( + SLoc.Invalid, new Identifier(exp.right.type.name), exp.left); castExp.env = exp.env; @@ -60,6 +63,7 @@ throw error(__LINE__, "Cannot make implicit cast"); auto castExp = new CastExp( + SLoc.Invalid, new Identifier(argType.name), arg); castExp.env = exp.exp.env; @@ -85,6 +89,7 @@ throw error(__LINE__, "Cannot make implicit cast between"); auto castExp = new CastExp( + SLoc.Invalid, new Identifier(expType.name), exp.exp); castExp.env = exp.exp.env; @@ -106,6 +111,7 @@ throw error(__LINE__, "Cannot make implicit cast"); auto castExp = new CastExp( + SLoc.Invalid, new Identifier(returnType.name), stmt.exp); castExp.env = stmt.exp.env; @@ -128,6 +134,7 @@ throw error(__LINE__, "Cannot make implicit cast"); auto castExp = new CastExp( + SLoc.Invalid, new Identifier(varType.name), decl.init); castExp.env = decl.init.env;
--- a/tools/AstPrinter.d Sun May 04 12:58:02 2008 +0200 +++ b/tools/AstPrinter.d Sun May 04 18:13:46 2008 +0200 @@ -6,16 +6,16 @@ ast.Stmt, ast.Exp; -import misc.DataSource; +import basic.SourceManager; class AstPrinter { const char[] tabType = " "; // 4 spaces - this(DataSource ds) + this(SourceManager sm) { - this.ds = ds; + this.sm = sm; } void print(Decl[] decls) @@ -112,8 +112,7 @@ break; case ExpType.IntegerLit: auto integetLit = cast(IntegerLit)exp; - auto t = integetLit.token; - print(t.get); + print(integetLit.get); break; case ExpType.Negate: auto negateExp = cast(NegateExp)exp; @@ -225,6 +224,6 @@ print(" "); } private: - DataSource ds; + SourceManager sm; char[] tabIndex; }
--- a/tools/DotPrinter.d Sun May 04 12:58:02 2008 +0200 +++ b/tools/DotPrinter.d Sun May 04 18:13:46 2008 +0200 @@ -7,9 +7,6 @@ ast.Stmt, ast.Exp; -import misc.DataSource, - lexer.Token; - class DotPrinter { this() @@ -116,7 +113,7 @@ case ExpType.IntegerLit: auto e = cast(IntegerLit)exp; - Stdout(id)(` [label="`)(text(e.token))(`"]`).newline; + Stdout(id)(` [label="`)(text(e.get))(`"]`).newline; break; case ExpType.Identifier: @@ -155,13 +152,11 @@ char[] text(Identifier identifier) { - auto t = identifier.token; - return t.get; + return identifier.get; } - char[] text(Token t) + char[] text(char[] t) { - return t.get; + return t; } -private: - DataSource ds; } +