view dmd/lexer.h @ 1638:0de4525a9ed6

Apply workaround for #395 by klickverbot.
author Christian Kamm <kamm incasoftware de>
date Mon, 08 Mar 2010 20:06:08 +0100
parents eae495e6ae8d
line wrap: on
line source

// Compiler implementation of the D programming language
// Copyright (c) 1999-2008 by Digital Mars
// All Rights Reserved
// written by Walter Bright
// License for redistribution is by either the Artistic License
// in artistic.txt, or the GNU General Public License in gnu.txt.
// See the included readme.txt for details.

#ifndef DMD_LEXER_H
#define DMD_LEXER_H

#ifdef __DMC__
#pragma once
#endif /* __DMC__ */

#include "root.h"
#include "mars.h"

struct StringTable;
struct Identifier;
struct Module;

/* Tokens:
	(	)
	[	]
	{	}
	<	>	<=	>=	==	!=	===	!==
	<<	>>	<<=	>>=	>>>	>>>=
	+	-	+=	-=
	*	/	%	*=	/=	%=
	&	| 	^	&=	|=	^=
	=	!	~
	++	--
	.	->	:	,
	?	&&	||

enum TOK

	// Other
	TOKlparen,	TOKrparen,
	TOKlbracket,	TOKrbracket,
	TOKlcurly,	TOKrcurly,
	TOKcolon,	TOKneg,
	TOKsemicolon,	TOKdotdotdot,
	TOKeof,		TOKcast,
	TOKnull,	TOKassert,
	TOKtrue,	TOKfalse,
	TOKarray,	TOKcall,
	TOKtype,	TOKthrow,
	TOKnew,		TOKdelete,
	TOKstar,	TOKsymoff,
	TOKvar,		TOKdotvar,
	TOKdotti,	TOKdotexp,
	TOKdottype,	TOKslice,
	TOKarraylength,	TOKversion,
	TOKmodule,	TOKdollar,
	TOKtemplate,	TOKdottd,
	TOKdeclaration,	TOKtypeof,
	TOKpragma,	TOKdsymbol,
	TOKtypeid,	TOKuadd,
	TOKnewanonclass, TOKcomment,
	TOKarrayliteral, TOKassocarrayliteral,

	// Operators
	TOKlt,		TOKgt,
	TOKle,		TOKge,
	TOKequal,	TOKnotequal,
	TOKidentity,	TOKnotidentity,
	TOKindex,	TOKis,

// 60
	// NCEG floating point compares
	// !<>=     <>    <>=    !>     !>=   !<     !<=   !<>

	TOKshl,		TOKshr,
	TOKshlass,	TOKshrass,
	TOKushr,	TOKushrass,
	TOKcat,		TOKcatass,	// ~ ~=
	TOKadd,		TOKmin,		TOKaddass,	TOKminass,
	TOKmul,		TOKdiv,		TOKmod,
	TOKmulass,	TOKdivass,	TOKmodass,
	TOKand,		TOKor,		TOKxor,
	TOKandass,	TOKorass,	TOKxorass,
	TOKassign,	TOKnot,		TOKtilde,
	TOKplusplus,	TOKminusminus,	TOKconstruct,	TOKblit,
	TOKdot,		TOKarrow,	TOKcomma,
	TOKquestion,	TOKandand,	TOKoror,

// 104
	// Numeric literals
	TOKint32v, TOKuns32v,
	TOKint64v, TOKuns64v,
	TOKfloat32v, TOKfloat64v, TOKfloat80v,
	TOKimaginary32v, TOKimaginary64v, TOKimaginary80v,

	// Char constants
	TOKcharv, TOKwcharv, TOKdcharv,

	// Leaf operators
	TOKidentifier,	TOKstring,
	TOKthis,	TOKsuper,
	TOKhalt,	TOKtuple,

	// Basic types
	TOKint8, TOKuns8,
	TOKint16, TOKuns16,
	TOKint32, TOKuns32,
	TOKint64, TOKuns64,
	TOKfloat32, TOKfloat64, TOKfloat80,
	TOKimaginary32, TOKimaginary64, TOKimaginary80,
	TOKcomplex32, TOKcomplex64, TOKcomplex80,
	TOKchar, TOKwchar, TOKdchar, TOKbit, TOKbool,
	TOKcent, TOKucent,

// 150
	// Aggregates
	TOKstruct, TOKclass, TOKinterface, TOKunion, TOKenum, TOKimport,
	TOKtypedef, TOKalias, TOKoverride, TOKdelegate, TOKfunction,

	TOKalign, TOKextern, TOKprivate, TOKprotected, TOKpublic, TOKexport,
	TOKstatic, /*TOKvirtual,*/ TOKfinal, TOKconst, TOKabstract, TOKvolatile,
	TOKdebug, TOKdeprecated, TOKin, TOKout, TOKinout, TOKlazy,
	TOKauto, TOKpackage, TOKmanifest,

	// Statements
	TOKif, TOKelse, TOKwhile, TOKfor, TOKdo, TOKswitch,
	TOKcase, TOKdefault, TOKbreak, TOKcontinue, TOKwith,
	TOKsynchronized, TOKreturn, TOKgoto, TOKtry, TOKcatch, TOKfinally,
	TOKasm, TOKforeach, TOKforeach_reverse,
	TOKon_scope_exit, TOKon_scope_failure, TOKon_scope_success,

	// Contracts
	TOKbody, TOKinvariant,

	// Testing

	// Added after 1.0
#if DMDV2

// LDC specific


#define BASIC_TYPES			\
	TOKwchar: case TOKdchar:		\
	case TOKbit: case TOKbool: case TOKchar:	\
	case TOKint8: case TOKuns8:		\
	case TOKint16: case TOKuns16:		\
	case TOKint32: case TOKuns32:		\
	case TOKint64: case TOKuns64:		\
	case TOKfloat32: case TOKfloat64: case TOKfloat80:		\
	case TOKimaginary32: case TOKimaginary64: case TOKimaginary80:	\
	case TOKcomplex32: case TOKcomplex64: case TOKcomplex80:	\
	case TOKvoid

#define BASIC_TYPES_X(t)					\
	TOKvoid:	 t = Type::tvoid;  goto LabelX;		\
	case TOKint8:	 t = Type::tint8;  goto LabelX;		\
	case TOKuns8:	 t = Type::tuns8;  goto LabelX;		\
	case TOKint16:	 t = Type::tint16; goto LabelX;		\
	case TOKuns16:	 t = Type::tuns16; goto LabelX;		\
	case TOKint32:	 t = Type::tint32; goto LabelX;		\
	case TOKuns32:	 t = Type::tuns32; goto LabelX;		\
	case TOKint64:	 t = Type::tint64; goto LabelX;		\
	case TOKuns64:	 t = Type::tuns64; goto LabelX;		\
	case TOKfloat32: t = Type::tfloat32; goto LabelX;	\
	case TOKfloat64: t = Type::tfloat64; goto LabelX;	\
	case TOKfloat80: t = Type::tfloat80; goto LabelX;	\
	case TOKimaginary32: t = Type::timaginary32; goto LabelX;	\
	case TOKimaginary64: t = Type::timaginary64; goto LabelX;	\
	case TOKimaginary80: t = Type::timaginary80; goto LabelX;	\
	case TOKcomplex32: t = Type::tcomplex32; goto LabelX;	\
	case TOKcomplex64: t = Type::tcomplex64; goto LabelX;	\
	case TOKcomplex80: t = Type::tcomplex80; goto LabelX;	\
	case TOKbit:	 t = Type::tbit;     goto LabelX;	\
	case TOKbool:	 t = Type::tbool;    goto LabelX;	\
	case TOKchar:	 t = Type::tchar;    goto LabelX;	\
	case TOKwchar:	 t = Type::twchar; goto LabelX;	\
	case TOKdchar:	 t = Type::tdchar; goto LabelX;	\

struct Token
    Token *next;
    unsigned char *ptr;		// pointer to first character of this token within buffer
    enum TOK value;
    unsigned char *blockComment; // doc comment string prior to this token
    unsigned char *lineComment;	 // doc comment for previous token
	// Integers
	d_int64	int64value;
	d_uns64	uns64value;

	// Floats
#ifdef IN_GCC
	// real_t float80value; // can't use this in a union!
	d_float80 float80value;

	{   unsigned char *ustring;	// UTF8 string
	    unsigned len;
	    unsigned char postfix;	// 'c', 'w', 'd'

	Identifier *ident;
#ifdef IN_GCC
    real_t float80value; // can't use this in a union!

    static const char *tochars[TOKMAX];
    static void *operator new(size_t sz);

    int isKeyword();
    void print();
    const char *toChars();
    static const char *toChars(enum TOK);

struct Lexer
    static StringTable stringtable;
    static OutBuffer stringbuffer;
    static Token *freelist;

    Loc loc;			// for error messages

    unsigned char *base;	// pointer to start of buffer
    unsigned char *end;		// past end of buffer
    unsigned char *p;		// current character
    Token token;
    Module *mod;
    int doDocComment;		// collect doc comment information
    int anyToken;		// !=0 means seen at least one token
    int commentToken;		// !=0 means comments are TOKcomment's

    Lexer(Module *mod,
	unsigned char *base, unsigned begoffset, unsigned endoffset,
	int doDocComment, int commentToken);

    static void initKeywords();
    static Identifier *idPool(const char *s);
    static Identifier *uniqueId(const char *s);
    static Identifier *uniqueId(const char *s, int num);

    TOK nextToken();
    TOK peekNext();
    void scan(Token *t);
    Token *peek(Token *t);
    Token *peekPastParen(Token *t);
    unsigned escapeSequence();
    TOK wysiwygStringConstant(Token *t, int tc);
    TOK hexStringConstant(Token *t);
#if DMDV2
    TOK delimitedStringConstant(Token *t);
    TOK tokenStringConstant(Token *t);
    TOK escapeStringConstant(Token *t, int wide);
    TOK charConstant(Token *t, int wide);
    void stringPostfix(Token *t);
    unsigned wchar(unsigned u);
    TOK number(Token *t);
    TOK inreal(Token *t);
    void error(const char *format, ...) IS_PRINTF(2);
    void error(Loc loc, const char *format, ...) IS_PRINTF(3);
    void pragma();
    unsigned decodeUTF();
    void getDocComment(Token *t, unsigned lineComment);

    static int isValidIdentifier(char *p);
    static unsigned char *combineComments(unsigned char *c1, unsigned char *c2);

#endif /* DMD_LEXER_H */