projects/ddmd: dmd/Lexer.d comparison

comparison dmd/Lexer.d @ 178:e3afd1303184

Many small bugs fixed Made all classes derive from TObject to detect memory leaks (functionality is disabled for now) Began work on overriding backend memory allocations (to avoid memory leaks)

author	korDen
date	Sun, 17 Oct 2010 07:42:00 +0400
parents	94b6033c07f3
children

comparison

equal deleted inserted replaced

-:1475fd394c9e
+:e3afd1303184
 	assert(0);		// should have been in table
 }
 	return true;
 }
-class Lexer
+import dmd.TObject;
+class Lexer : TObject
 {
 Loc loc;			// for error messages
 ubyte* base;	// pointer to start of buffer
 ubyte* end;		// past end of buffer
 int anyToken;		// !=0 means seen at least one token
 int commentToken;		// !=0 means comments are TOKcomment's
 this(Module mod, ubyte* base, uint begoffset, uint endoffset, int doDocComment, int commentToken)
 	{
+		register();
 		loc = Loc(mod, 1);
 		memset(&token,0,token.sizeof);
 		this.base = base;
 		this.end  = base + endoffset;
 		p = base + begoffset;
 		this.mod = mod;
 		if (p[0] == '#' && p[1] =='!')
 		{
 			p += 2;
 			while (1)
 			{
 				ubyte c = *p;
 				switch (c)
 				{
 				case '\n':
 					p++;
 	static __gshared ubyte cmtable[256];
 	enum CMoctal =	0x1;
 	enum  CMhex =	0x2;
 	enum  CMidchar =	0x4;
 	ubyte isoctal (ubyte c) { return cmtable[c] & CMoctal; }
 	ubyte ishex   (ubyte c) { return cmtable[c] & CMhex; }
 	ubyte isidchar(ubyte c) { return cmtable[c] & CMidchar; }
 	static void cmtable_init()
 				cmtable[c] |= CMhex;
 			if (isalnum(c) || c == '_')
 				cmtable[c] |= CMidchar;
 		}
 	}
-	static StringTable stringtable()
+	static ref StringTable stringtable()
 	{
 		return global.stringtable;
 	}
 	static OutBuffer stringbuffer()
 	{
 		return global.stringbuffer;
 	}
 static void initKeywords()
 	{
 		uint nkeywords = keywords.length;
 		if (global.params.Dversion == 1)
 			nkeywords -= 2;
 		cmtable_init();
 		for (uint u = 0; u < nkeywords; u++)
 		{
 			//printf("keyword[%d] = '%.*s'\n",u, keywords[u].name);
 			string s = keywords[u].name;
 			TOK v = keywords[u].value;
-			StringValue* sv = stringtable.insert(s);
+			Object* sv = stringtable.insert(s);
-			sv.ptrvalue = cast(void*) new Identifier(sv.lstring.string_, v);
+			*sv = new Identifier(s, v);
 			//printf("tochars[%d] = '%s'\n",v, s);
 			Token.tochars[v] = s;
 		}
 		Token.tochars[TOK.TOKorass]		= "|=";
 		Token.tochars[TOK.TOKidentifier]	= "identifier";
 		Token.tochars[TOK.TOKat]		= "@";
 Token.tochars[TOK.TOKpow]		= "^^";
 Token.tochars[TOK.TOKpowass]		= "^^=";
 		 // For debugging
 		Token.tochars[TOKerror]		= "error";
 		Token.tochars[TOK.TOKdotexp]		= "dotexp";
 		Token.tochars[TOK.TOKdotti]		= "dotti";
 		Token.tochars[TOK.TOKdotvar]		= "dotvar";
 		Token.tochars[TOK.TOKon_scope_failure]	= "scope(failure)";
 	}
 static Identifier idPool(string s)
 	{
-		StringValue* sv = stringtable.update(s);
+		Object* sv = stringtable.update(s);
-		Identifier id = cast(Identifier) sv.ptrvalue;
+		Identifier id = cast(Identifier) *sv;
 		if (id is null)
 		{
-			id = new Identifier(sv.lstring.string_, TOK.TOKidentifier);
+			id = new Identifier(s, TOK.TOKidentifier);
-			sv.ptrvalue = cast(void*)id;
+			*sv = id;
 		}
 		return id;
 	}
 			            p = s;
 		            }
 		            break;
 		        }
-		        StringValue *sv = stringtable.update((cast(immutable(char)*)t.ptr)[0.. p - t.ptr]);
+auto s = cast(string)(t.ptr[0.. p - t.ptr]);
-		        Identifier id = cast(Identifier) sv.ptrvalue;
+		        Object* sv = stringtable.update(s);
+		        Identifier id = cast(Identifier) *sv;
 				if (id is null)
-				{   id = new Identifier(sv.lstring.string_, TOK.TOKidentifier);
+				{
-					sv.ptrvalue = cast(void*)id;
+				    id = new Identifier(s, TOK.TOKidentifier);
+					*sv = id;
 				}
 				t.ident = id;
 				t.value = cast(TOK) id.value;
 				anyToken = 1;
 				if (*t.ptr == '_')	// if special identifier token
 						{   // if /++ but not /++/
 							getDocComment(t, lastLine == linnum);
 						}
 						continue;
 					}
 					default:
 						break;	///
 				}
 				t.value = TOK.TOKdiv;
 				return;
 					t.value = TOK.TOKcatass;		// ~=
 				}
 				else
 					t.value = TOK.TOKtilde;		// ~
 				return;
 version(DMDV2) {
 	    case '^':
 		p++;
 		if (*p == '^')
 		{   p++;
 						t.value = TOK.TOKmulass;
 					} else {
 						t.value = TOK.TOKmul;
 					}
 					return;
 				case '%':
 					p++;
 					if (*p == '=') {
 						p++;
 						t.value = TOK.TOKmodass;
 				ndigits = 2;
 			Lhex:
 				p++;
 				c = *p;
 				if (ishex(cast(ubyte)c))
 				{
 					uint v;
 					n = 0;
 					v = 0;
 					while (1)
 					switch (*p)
 					{
 					case ';':
 						c = HtmlNamedEntity(idstart, p - idstart);
 						if (c == ~0)
 						{
 							error("unnamed character entity &%s;", idstart[0..(p - idstart)]);
 							c = ' ';
 						}
 						p++;
 						break;
 				c = '\\';
 				break;
 			default:
 				if (isoctal(cast(ubyte)c))
 				{
 					uint v;
 					n = 0;
 					v = 0;
 					do
 					}
 					break;
 			}
 			stringbuffer.writeByte(c);
 		}
 		assert(false);
 	}
 	/**************************************
 	 * Lex hex strings:
 					t.postfix = 0;
 					return TOKstring;
 				case '"':
 					if (n & 1)
 					{
 						error("odd number (%d) of hex characters in hex string", n);
 						stringbuffer.writeByte(v);
 					}
 					t.len = stringbuffer.offset;
 					stringbuffer.writeByte(0);
 						goto Lnextline;
 					}
 					break;
 			}
 			if (delimleft == 0)
 			{
 				delimleft = c;
 				nest = 1;
 				nestcount = 1;
 				if (c == '(')
 					delimright = ')';
 				else if (c == '[')
 					delimright = ']';
 				else if (c == '<')
 					delimright = '>';
 				else if (isalpha(c) || c == '_' || (c >= 0x80 && isUniAlpha(c)))
 				{
 					// Start of identifier; must be a heredoc
 					Token t2;
 					p--;
 					scan(&t2);		// read in heredoc identifier
 					if (t2.value != TOKidentifier)
 					{
 						error("identifier expected for heredoc, not %s", t2.toChars());
 						delimright = c;
 					}
 					else
 					{
 						hereid = t2.ident;
 						//printf("hereid = '%s'\n", hereid.toChars());
 						blankrol = 1;
 					}
 					nest = 0;
 				}
 				else
 				{
 					delimright = c;
 					nest = 0;
 					if (isspace(c))
 						error("delimiter cannot be whitespace");
 				}
 			}
 			else
 			{
 				if (blankrol)
 				{
 					error("heredoc rest of line should be blank");
 					blankrol = 0;
 					continue;
 				}
 				if (nest == 1)
 					}
 				}
 				else if (c == delimright)
 					goto Ldone;
 				if (startline && isalpha(c) && hereid)
 				{
 					Token t2;
 					ubyte* psave = p;
 					p--;
 					scan(&t2);		// read in possible heredoc identifier
 					//printf("endid = '%s'\n", t2.ident.toChars());
 					if (t2.value == TOKidentifier && t2.ident.equals(hereid))
 					{
 						/* should check that rest of line is blank
 						 */
 						goto Ldone;
 					}
 					p = psave;
 		uint nest = 1;
 		Loc start = loc;
 		ubyte* pstart = ++p;
 		while (true)
 		{
 			Token tok;
 			scan(&tok);
 			switch (tok.value)
 			{
 					}
 					break;
 			}
 			stringbuffer.writeByte(c);
 		}
 		assert(false);
 	}
 TOK charConstant(Token* t, int wide)
 	{
 				t.uns64value = c;
 				break;
 		}
 		if (*p != '\'')
 		{
 			error("unterminated character constant");
 			return tk;
 		}
 		p++;
 		return tk;
 uint wchar_(uint u)
 	{
 		assert(false);
 	}
 	/**************************************
 	 * Read in a number.
 	 * If it's an integer, store it in tok.TKutok.Vlong.
 	 *	integers can be decimal, octal or hex
 	 *	Handle the suffixes U, UL, LU, L, etc.
 			FLAGS_undefined = 0,
 			FLAGS_decimal  = 1,		// decimal
 			FLAGS_unsigned = 2,		// u or U suffix
 			FLAGS_long     = 4,		// l or L suffix
 		};
 		FLAGS flags = FLAGS.FLAGS_decimal;
 		int i;
 		int base;
 		uint c;
 						break;
 					case 9:
 						dblstate = 1;
 						if (c == 'X' || c == 'x')
 						{
 							hex++;
 							break;
 						}
 					case 1:			// digits to left of .
 					case 3:			// digits to right of .
 						}
 						break;
 					case 2:			// no more digits to left of .
 						if (c == '.')
 						{
 							dblstate++;
 							break;
 						}
 					case 4:			// no more digits to right of .
 						if ((c == 'E' || c == 'e') ||
 							hex && (c == 'P' || c == 'p'))
 						{
 							dblstate = 5;
 							hex = 0;	// exponent is always decimal
 							break;
 						}
 						if (hex)
 					result = TOKimaginary80v;
 					break;
 				default:
 			}
 		}
 	version (Windows) { ///&& __DMC__
 		__locale_decpoint = save;
 	}
 		if (errno == ERANGE)
 			error("number is not representable");
 				fatal();
 		}
 		global.errors++;
 	}
 	/*********************************************
 	 * Do pragma.
 	 * Currently, the only pragma supported is:
 	 *	#line linnum [filespec]
 	 */
 					if (filespec)
 						goto Lerr;
 					stringbuffer.reset();
 					p++;
 					while (1)
 					{
 						uint c;
 						c = *p;
 						switch (c)
 						{
 								p++;
 								break;
 							default:
 								if (c & 0x80)
 								{
 									uint u = decodeUTF();
 									if (u == PS || u == LS)
 										goto Lerr;
 								}
 								stringbuffer.writeByte(c);
 					}
 					continue;
 				default:
 					if (*p & 0x80)
 					{
 						uint u = decodeUTF();
 						if (u == PS || u == LS)
 							goto Lnewline;
 					}
 					goto Lerr;
 	static string combineComments(string c1, string c2)
 	{
 		//printf("Lexer::combineComments('%s', '%s')\n", c1, c2);
 		string c = c2;
 		if (c1)
 		{
 			c = c1;
 			if (c2)
 			{
 	static string combineComments(const(char)[] c1, const(char)[] c2)
 	{
 		//writef("Lexer.combineComments('%s', '%s')\n", c1, c2);
 		char[] c = cast(char[]) c2;
 		if (c1 !is null)
 		{
 			c = cast(char[]) c1;
 			if (c2 !is null)
 			{
 				c[0..len1] = c1[];
 				c[len1++] = '\n';
 				c[len1 .. len1 + c2.length] = c2[];
 			}
 		}
 		return cast(string)c;
 	}
 }

Mercurial > projects > ddmd

comparison dmd/Lexer.d @ 178:e3afd1303184