projects/dil: trunk/src/Lexer.d annotate

annotate trunk/src/Lexer.d @ 53:1786c2825491

- Fixed scanner for hex string literals. Terminating string with 0. Relocated some code to the bottom.

author	aziz
date	Wed, 27 Jun 2007 17:50:01 +0000
parents	f65a83c27638
children	e55bd2270f94

rev	line source
0 8ba2570de175 Initial import. aziz parents: diff changeset	1 /++
8ba2570de175 Initial import. aziz parents: diff changeset	2 Author: Aziz Köksal
8ba2570de175 Initial import. aziz parents: diff changeset	3 License: GPL2
8ba2570de175 Initial import. aziz parents: diff changeset	4 +/
8ba2570de175 Initial import. aziz parents: diff changeset	5 module Lexer;
3 4bbce78bfb1e - Added TOK enum. aziz parents: 2 diff changeset	6 import Token;
28 3a9daccf7d96 - Added table for identifiers to Lexer. aziz parents: 27 diff changeset	7 import Keywords;
3a9daccf7d96 - Added table for identifiers to Lexer. aziz parents: 27 diff changeset	8 import Identifier;
2 81c6cc33f5c8 - Initializing ptable with a precomputed array literal. aziz parents: 1 diff changeset	9 import std.stdio;
4 92df59b1ec4a - Started implementation of scan(). aziz parents: 3 diff changeset	10 import std.utf;
92df59b1ec4a - Started implementation of scan(). aziz parents: 3 diff changeset	11 import std.uni;
30 426767b94635 - Added code for parsing the '#line' special token. aziz parents: 29 diff changeset	12 import std.conv;
0 8ba2570de175 Initial import. aziz parents: diff changeset	13
8ba2570de175 Initial import. aziz parents: diff changeset	14 /// ASCII character properties table.
2 81c6cc33f5c8 - Initializing ptable with a precomputed array literal. aziz parents: 1 diff changeset	15 static const int ptable[256] = [
52 f65a83c27638 - Fixed the raw string literal scanner. Newlines weren't copied to the buffer. Converting LS and PS to '\n' as well. aziz parents: 51 diff changeset	16 0, 0, 0, 0, 0, 0, 0, 0, 0,32, 0,32,32, 0, 0, 0,
f65a83c27638 - Fixed the raw string literal scanner. Newlines weren't copied to the buffer. Converting LS and PS to '\n' as well. aziz parents: 51 diff changeset	17 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
45 cc84b9ca9d0a - Implemented escape sequence scanner. aziz parents: 44 diff changeset	18 32, 0, 0x2200, 0, 0, 0, 0, 0x2700, 0, 0, 0, 0, 0, 0, 0, 0,
cc84b9ca9d0a - Implemented escape sequence scanner. aziz parents: 44 diff changeset	19 7, 7, 7, 7, 7, 7, 7, 7, 6, 6, 0, 0, 0, 0, 0, 0x3f00,
13 e5211758b63c - Added isidbeg() function. aziz parents: 12 diff changeset	20 0,12,12,12,12,12,12, 8, 8, 8, 8, 8, 8, 8, 8, 8,
45 cc84b9ca9d0a - Implemented escape sequence scanner. aziz parents: 44 diff changeset	21 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 0, 0x5c00, 0, 0,16,
cc84b9ca9d0a - Implemented escape sequence scanner. aziz parents: 44 diff changeset	22 0, 0x70c, 0x80c,12,12,12, 0xc0c, 8, 8, 8, 8, 8, 8, 8, 0xa08, 8,
cc84b9ca9d0a - Implemented escape sequence scanner. aziz parents: 44 diff changeset	23 8, 8, 0xd08, 8, 0x908, 8, 0xb08, 8, 8, 8, 8, 0, 0, 0, 0, 0,
13 e5211758b63c - Added isidbeg() function. aziz parents: 12 diff changeset	24 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
e5211758b63c - Added isidbeg() function. aziz parents: 12 diff changeset	25 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
e5211758b63c - Added isidbeg() function. aziz parents: 12 diff changeset	26 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
e5211758b63c - Added isidbeg() function. aziz parents: 12 diff changeset	27 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
e5211758b63c - Added isidbeg() function. aziz parents: 12 diff changeset	28 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
e5211758b63c - Added isidbeg() function. aziz parents: 12 diff changeset	29 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
e5211758b63c - Added isidbeg() function. aziz parents: 12 diff changeset	30 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
e5211758b63c - Added isidbeg() function. aziz parents: 12 diff changeset	31 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
2 81c6cc33f5c8 - Initializing ptable with a precomputed array literal. aziz parents: 1 diff changeset	32 ];
0 8ba2570de175 Initial import. aziz parents: diff changeset	33
8ba2570de175 Initial import. aziz parents: diff changeset	34 enum CProperty
8ba2570de175 Initial import. aziz parents: diff changeset	35 {
1 f3cd3bfde4ba - Corrected some errors to make the file compile. aziz parents: 0 diff changeset	36 Octal = 1,
0 8ba2570de175 Initial import. aziz parents: diff changeset	37 Digit = 1<<1,
8ba2570de175 Initial import. aziz parents: diff changeset	38 Hex = 1<<2,
8ba2570de175 Initial import. aziz parents: diff changeset	39 Alpha = 1<<3,
13 e5211758b63c - Added isidbeg() function. aziz parents: 12 diff changeset	40 Underscore = 1<<4,
16 476e8e55c1d4 - Added Whitespace to the character properties table. aziz parents: 15 diff changeset	41 Whitespace = 1<<5
0 8ba2570de175 Initial import. aziz parents: diff changeset	42 }
8ba2570de175 Initial import. aziz parents: diff changeset	43
45 cc84b9ca9d0a - Implemented escape sequence scanner. aziz parents: 44 diff changeset	44 const uint EVMask = 0xFF00; // Bit mask for escape value
cc84b9ca9d0a - Implemented escape sequence scanner. aziz parents: 44 diff changeset	45
13 e5211758b63c - Added isidbeg() function. aziz parents: 12 diff changeset	46 private alias CProperty CP;
e5211758b63c - Added isidbeg() function. aziz parents: 12 diff changeset	47 int isoctal(char c) { return ptable[c] & CP.Octal; }
e5211758b63c - Added isidbeg() function. aziz parents: 12 diff changeset	48 int isdigit(char c) { return ptable[c] & CP.Digit; }
e5211758b63c - Added isidbeg() function. aziz parents: 12 diff changeset	49 int ishexad(char c) { return ptable[c] & CP.Hex; }
e5211758b63c - Added isidbeg() function. aziz parents: 12 diff changeset	50 int isalpha(char c) { return ptable[c] & CP.Alpha; }
e5211758b63c - Added isidbeg() function. aziz parents: 12 diff changeset	51 int isalnum(char c) { return ptable[c] & (CP.Alpha \| CP.Digit); }
e5211758b63c - Added isidbeg() function. aziz parents: 12 diff changeset	52 int isidbeg(char c) { return ptable[c] & (CP.Alpha \| CP.Underscore); }
e5211758b63c - Added isidbeg() function. aziz parents: 12 diff changeset	53 int isident(char c) { return ptable[c] & (CP.Alpha \| CP.Underscore \| CP.Digit); }
16 476e8e55c1d4 - Added Whitespace to the character properties table. aziz parents: 15 diff changeset	54 int isspace(char c) { return ptable[c] & CP.Whitespace; }
45 cc84b9ca9d0a - Implemented escape sequence scanner. aziz parents: 44 diff changeset	55 int char2ev(char c) { return ptable[c] >> 8; /(ptable[c] & EVMask) >> 8;/ }
13 e5211758b63c - Added isidbeg() function. aziz parents: 12 diff changeset	56
e5211758b63c - Added isidbeg() function. aziz parents: 12 diff changeset	57 version(gen_ptable)
0 8ba2570de175 Initial import. aziz parents: diff changeset	58 static this()
8ba2570de175 Initial import. aziz parents: diff changeset	59 {
45 cc84b9ca9d0a - Implemented escape sequence scanner. aziz parents: 44 diff changeset	60 alias ptable p;
2 81c6cc33f5c8 - Initializing ptable with a precomputed array literal. aziz parents: 1 diff changeset	61 // Initialize character properties table.
45 cc84b9ca9d0a - Implemented escape sequence scanner. aziz parents: 44 diff changeset	62 for (int i; i < p.length; ++i)
0 8ba2570de175 Initial import. aziz parents: diff changeset	63 {
45 cc84b9ca9d0a - Implemented escape sequence scanner. aziz parents: 44 diff changeset	64 p[i] = 0;
0 8ba2570de175 Initial import. aziz parents: diff changeset	65 if ('0' <= i && i <= '7')
45 cc84b9ca9d0a - Implemented escape sequence scanner. aziz parents: 44 diff changeset	66 p[i] \|= CP.Octal;
0 8ba2570de175 Initial import. aziz parents: diff changeset	67 if ('0' <= i && i <= '9')
45 cc84b9ca9d0a - Implemented escape sequence scanner. aziz parents: 44 diff changeset	68 p[i] \|= CP.Digit;
0 8ba2570de175 Initial import. aziz parents: diff changeset	69 if (isdigit(i) \|\| 'a' <= i && i <= 'f' \|\| 'A' <= i && i <= 'F')
45 cc84b9ca9d0a - Implemented escape sequence scanner. aziz parents: 44 diff changeset	70 p[i] \|= CP.Hex;
0 8ba2570de175 Initial import. aziz parents: diff changeset	71 if ('a' <= i && i <= 'z' \|\| 'A' <= i && i <= 'Z')
45 cc84b9ca9d0a - Implemented escape sequence scanner. aziz parents: 44 diff changeset	72 p[i] \|= CP.Alpha;
13 e5211758b63c - Added isidbeg() function. aziz parents: 12 diff changeset	73 if (i == '_')
45 cc84b9ca9d0a - Implemented escape sequence scanner. aziz parents: 44 diff changeset	74 p[i] \|= CP.Underscore;
44 5055947e0f98 - Specific operators and comments can be formatted with CSS now. aziz parents: 43 diff changeset	75 if (i == ' ' \|\| i == '\t' \|\| i == '\v' \|\| i == '\f')
45 cc84b9ca9d0a - Implemented escape sequence scanner. aziz parents: 44 diff changeset	76 p[i] \|= CP.Whitespace;
0 8ba2570de175 Initial import. aziz parents: diff changeset	77 }
45 cc84b9ca9d0a - Implemented escape sequence scanner. aziz parents: 44 diff changeset	78 // Store escape sequence values in second byte.
cc84b9ca9d0a - Implemented escape sequence scanner. aziz parents: 44 diff changeset	79 assert(CProperty.max <= ubyte.max, "character property flags and escape value byte overlap.");
cc84b9ca9d0a - Implemented escape sequence scanner. aziz parents: 44 diff changeset	80 p['\''] \|= 39 << 8;
cc84b9ca9d0a - Implemented escape sequence scanner. aziz parents: 44 diff changeset	81 p['"'] \|= 34 << 8;
cc84b9ca9d0a - Implemented escape sequence scanner. aziz parents: 44 diff changeset	82 p['?'] \|= 63 << 8;
52 f65a83c27638 - Fixed the raw string literal scanner. Newlines weren't copied to the buffer. Converting LS and PS to '\n' as well. aziz parents: 51 diff changeset	83 p['\\'] \|= 92 << 8;
45 cc84b9ca9d0a - Implemented escape sequence scanner. aziz parents: 44 diff changeset	84 p['a'] \|= 7 << 8;
cc84b9ca9d0a - Implemented escape sequence scanner. aziz parents: 44 diff changeset	85 p['b'] \|= 8 << 8;
cc84b9ca9d0a - Implemented escape sequence scanner. aziz parents: 44 diff changeset	86 p['f'] \|= 12 << 8;
cc84b9ca9d0a - Implemented escape sequence scanner. aziz parents: 44 diff changeset	87 p['n'] \|= 10 << 8;
cc84b9ca9d0a - Implemented escape sequence scanner. aziz parents: 44 diff changeset	88 p['r'] \|= 13 << 8;
cc84b9ca9d0a - Implemented escape sequence scanner. aziz parents: 44 diff changeset	89 p['t'] \|= 9 << 8;
cc84b9ca9d0a - Implemented escape sequence scanner. aziz parents: 44 diff changeset	90 p['v'] \|= 11 << 8;
2 81c6cc33f5c8 - Initializing ptable with a precomputed array literal. aziz parents: 1 diff changeset	91 // Print a formatted array literal.
81c6cc33f5c8 - Initializing ptable with a precomputed array literal. aziz parents: 1 diff changeset	92 char[] array = "[\n";
45 cc84b9ca9d0a - Implemented escape sequence scanner. aziz parents: 44 diff changeset	93 for (int i; i < p.length; ++i)
2 81c6cc33f5c8 - Initializing ptable with a precomputed array literal. aziz parents: 1 diff changeset	94 {
45 cc84b9ca9d0a - Implemented escape sequence scanner. aziz parents: 44 diff changeset	95 int c = p[i];
cc84b9ca9d0a - Implemented escape sequence scanner. aziz parents: 44 diff changeset	96 array ~= std.string.format(c>255?" 0x%x,":"%2d,", c, ((i+1) % 16) ? "":"\n");
2 81c6cc33f5c8 - Initializing ptable with a precomputed array literal. aziz parents: 1 diff changeset	97 }
13 e5211758b63c - Added isidbeg() function. aziz parents: 12 diff changeset	98 array[$-2..$] = "\n]";
2 81c6cc33f5c8 - Initializing ptable with a precomputed array literal. aziz parents: 1 diff changeset	99 writefln(array);
0 8ba2570de175 Initial import. aziz parents: diff changeset	100 }
8ba2570de175 Initial import. aziz parents: diff changeset	101
11 dffcdaa7c47a - Added Unicode line and paragraph separators. aziz parents: 10 diff changeset	102 const char[3] LS = \u2028;
dffcdaa7c47a - Added Unicode line and paragraph separators. aziz parents: 10 diff changeset	103 const char[3] PS = \u2029;
dffcdaa7c47a - Added Unicode line and paragraph separators. aziz parents: 10 diff changeset	104
18 c48d2125f1e2 - Moved code for scanning character literals to separate function. aziz parents: 17 diff changeset	105 const dchar LSd = 0x2028;
c48d2125f1e2 - Moved code for scanning character literals to separate function. aziz parents: 17 diff changeset	106 const dchar PSd = 0x2029;
c48d2125f1e2 - Moved code for scanning character literals to separate function. aziz parents: 17 diff changeset	107
32 d7011daa4740 - Added missing commas after the items in the messages table. aziz parents: 31 diff changeset	108 const uint _Z_ = 26; /// Control+Z
d7011daa4740 - Added missing commas after the items in the messages table. aziz parents: 31 diff changeset	109
18 c48d2125f1e2 - Moved code for scanning character literals to separate function. aziz parents: 17 diff changeset	110 /// Index into table of error messages.
c48d2125f1e2 - Moved code for scanning character literals to separate function. aziz parents: 17 diff changeset	111 enum MID
c48d2125f1e2 - Moved code for scanning character literals to separate function. aziz parents: 17 diff changeset	112 {
48 c2e0e0269c28 - Added code for scanning escape string literals. aziz parents: 47 diff changeset	113 InvalidUnicodeCharacter,
49 7f0fa15dcffc - Renamed function. aziz parents: 48 diff changeset	114 InvalidUTF8Sequence,
48 c2e0e0269c28 - Added code for scanning escape string literals. aziz parents: 47 diff changeset	115 // ''
18 c48d2125f1e2 - Moved code for scanning character literals to separate function. aziz parents: 17 diff changeset	116 UnterminatedCharacterLiteral,
30 426767b94635 - Added code for parsing the '#line' special token. aziz parents: 29 diff changeset	117 EmptyCharacterLiteral,
31 94f09f4e988e - Added struct for strings to Token with 'pf' = postfix. aziz parents: 30 diff changeset	118 // #line
51 cadd2bfe686c - Displaying error messages in XML. aziz parents: 50 diff changeset	119 ExpectedIdentifierSTLine,
cadd2bfe686c - Displaying error messages in XML. aziz parents: 50 diff changeset	120 ExpectedNormalStringLiteral,
cadd2bfe686c - Displaying error messages in XML. aziz parents: 50 diff changeset	121 ExpectedNumberAfterSTLine,
30 426767b94635 - Added code for parsing the '#line' special token. aziz parents: 29 diff changeset	122 NewlineInSpecialToken,
31 94f09f4e988e - Added struct for strings to Token with 'pf' = postfix. aziz parents: 30 diff changeset	123 UnterminatedSpecialToken,
47 8aa37a78937b - Properly implemented scanner for normal string literals. aziz parents: 46 diff changeset	124 // ""
8aa37a78937b - Properly implemented scanner for normal string literals. aziz parents: 46 diff changeset	125 UnterminatedString,
31 94f09f4e988e - Added struct for strings to Token with 'pf' = postfix. aziz parents: 30 diff changeset	126 // x""
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix. aziz parents: 30 diff changeset	127 NonHexCharInHexString,
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix. aziz parents: 30 diff changeset	128 OddNumberOfDigitsInHexString,
32 d7011daa4740 - Added missing commas after the items in the messages table. aziz parents: 31 diff changeset	129 UnterminatedHexString,
d7011daa4740 - Added missing commas after the items in the messages table. aziz parents: 31 diff changeset	130 // /* */ /+ +/
d7011daa4740 - Added missing commas after the items in the messages table. aziz parents: 31 diff changeset	131 UnterminatedBlockComment,
33 cf3047cf3cd2 - Added code for parsing back quote and raw strings. aziz parents: 32 diff changeset	132 UnterminatedNestedComment,
cf3047cf3cd2 - Added code for parsing back quote and raw strings. aziz parents: 32 diff changeset	133 // `` r""
cf3047cf3cd2 - Added code for parsing back quote and raw strings. aziz parents: 32 diff changeset	134 UnterminatedRawString,
cf3047cf3cd2 - Added code for parsing back quote and raw strings. aziz parents: 32 diff changeset	135 UnterminatedBackQuoteString,
45 cc84b9ca9d0a - Implemented escape sequence scanner. aziz parents: 44 diff changeset	136 // \x \u \U
cc84b9ca9d0a - Implemented escape sequence scanner. aziz parents: 44 diff changeset	137 UndefinedEscapeSequence,
cc84b9ca9d0a - Implemented escape sequence scanner. aziz parents: 44 diff changeset	138 InsufficientHexDigits,
cc84b9ca9d0a - Implemented escape sequence scanner. aziz parents: 44 diff changeset	139 // \&[a-zA-Z][a-zA-Z0-9]+;
cc84b9ca9d0a - Implemented escape sequence scanner. aziz parents: 44 diff changeset	140 UnterminatedHTMLEntity,
cc84b9ca9d0a - Implemented escape sequence scanner. aziz parents: 44 diff changeset	141 InvalidBeginHTMLEntity,
18 c48d2125f1e2 - Moved code for scanning character literals to separate function. aziz parents: 17 diff changeset	142 }
c48d2125f1e2 - Moved code for scanning character literals to separate function. aziz parents: 17 diff changeset	143
31 94f09f4e988e - Added struct for strings to Token with 'pf' = postfix. aziz parents: 30 diff changeset	144 string[] messages = [
48 c2e0e0269c28 - Added code for scanning escape string literals. aziz parents: 47 diff changeset	145 "invalid Unicode character.",
49 7f0fa15dcffc - Renamed function. aziz parents: 48 diff changeset	146 "invalid UTF-8 sequence.",
48 c2e0e0269c28 - Added code for scanning escape string literals. aziz parents: 47 diff changeset	147 // ''
32 d7011daa4740 - Added missing commas after the items in the messages table. aziz parents: 31 diff changeset	148 "unterminated character literal.",
30 426767b94635 - Added code for parsing the '#line' special token. aziz parents: 29 diff changeset	149 "empty character literal.",
31 94f09f4e988e - Added struct for strings to Token with 'pf' = postfix. aziz parents: 30 diff changeset	150 // #line
32 d7011daa4740 - Added missing commas after the items in the messages table. aziz parents: 31 diff changeset	151 "expected 'line' after '#'.",
51 cadd2bfe686c - Displaying error messages in XML. aziz parents: 50 diff changeset	152 `the filespec must be defined in a double quote string literal (e.g. "filespec".)`,
cadd2bfe686c - Displaying error messages in XML. aziz parents: 50 diff changeset	153 "positive integer expected after #line",
32 d7011daa4740 - Added missing commas after the items in the messages table. aziz parents: 31 diff changeset	154 "newline not allowed inside special token.",
51 cadd2bfe686c - Displaying error messages in XML. aziz parents: 50 diff changeset	155 "expected a terminating newline after special token.",
47 8aa37a78937b - Properly implemented scanner for normal string literals. aziz parents: 46 diff changeset	156 // ""
8aa37a78937b - Properly implemented scanner for normal string literals. aziz parents: 46 diff changeset	157 "unterminated string literal.",
31 94f09f4e988e - Added struct for strings to Token with 'pf' = postfix. aziz parents: 30 diff changeset	158 // x""
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix. aziz parents: 30 diff changeset	159 "non-hex character '{1}' found in hex string.",
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix. aziz parents: 30 diff changeset	160 "odd number of hex digits in hex string.",
32 d7011daa4740 - Added missing commas after the items in the messages table. aziz parents: 31 diff changeset	161 "unterminated hex string.",
d7011daa4740 - Added missing commas after the items in the messages table. aziz parents: 31 diff changeset	162 // /* */ /+ +/
d7011daa4740 - Added missing commas after the items in the messages table. aziz parents: 31 diff changeset	163 "unterminated block comment (/* */).",
33 cf3047cf3cd2 - Added code for parsing back quote and raw strings. aziz parents: 32 diff changeset	164 "unterminated nested comment (/+ +/).",
cf3047cf3cd2 - Added code for parsing back quote and raw strings. aziz parents: 32 diff changeset	165 // `` r""
cf3047cf3cd2 - Added code for parsing back quote and raw strings. aziz parents: 32 diff changeset	166 "unterminated raw string.",
cf3047cf3cd2 - Added code for parsing back quote and raw strings. aziz parents: 32 diff changeset	167 "unterminated back quote string.",
45 cc84b9ca9d0a - Implemented escape sequence scanner. aziz parents: 44 diff changeset	168 // \x \u \U
cc84b9ca9d0a - Implemented escape sequence scanner. aziz parents: 44 diff changeset	169 "found undefined escape sequence.",
cc84b9ca9d0a - Implemented escape sequence scanner. aziz parents: 44 diff changeset	170 "insufficient number of hex digits in escape sequence.",
cc84b9ca9d0a - Implemented escape sequence scanner. aziz parents: 44 diff changeset	171 // \&[a-zA-Z][a-zA-Z0-9]+;
cc84b9ca9d0a - Implemented escape sequence scanner. aziz parents: 44 diff changeset	172 "unterminated html entity.",
cc84b9ca9d0a - Implemented escape sequence scanner. aziz parents: 44 diff changeset	173 "html entities must begin with a letter.",
18 c48d2125f1e2 - Moved code for scanning character literals to separate function. aziz parents: 17 diff changeset	174 ];
c48d2125f1e2 - Moved code for scanning character literals to separate function. aziz parents: 17 diff changeset	175
c48d2125f1e2 - Moved code for scanning character literals to separate function. aziz parents: 17 diff changeset	176 class Problem
c48d2125f1e2 - Moved code for scanning character literals to separate function. aziz parents: 17 diff changeset	177 {
c48d2125f1e2 - Moved code for scanning character literals to separate function. aziz parents: 17 diff changeset	178 enum Type
c48d2125f1e2 - Moved code for scanning character literals to separate function. aziz parents: 17 diff changeset	179 {
c48d2125f1e2 - Moved code for scanning character literals to separate function. aziz parents: 17 diff changeset	180 Lexer,
c48d2125f1e2 - Moved code for scanning character literals to separate function. aziz parents: 17 diff changeset	181 Parser,
c48d2125f1e2 - Moved code for scanning character literals to separate function. aziz parents: 17 diff changeset	182 Semantic
c48d2125f1e2 - Moved code for scanning character literals to separate function. aziz parents: 17 diff changeset	183 }
c48d2125f1e2 - Moved code for scanning character literals to separate function. aziz parents: 17 diff changeset	184
c48d2125f1e2 - Moved code for scanning character literals to separate function. aziz parents: 17 diff changeset	185 MID id;
c48d2125f1e2 - Moved code for scanning character literals to separate function. aziz parents: 17 diff changeset	186 Type type;
c48d2125f1e2 - Moved code for scanning character literals to separate function. aziz parents: 17 diff changeset	187 uint loc;
c48d2125f1e2 - Moved code for scanning character literals to separate function. aziz parents: 17 diff changeset	188 this(Type type, MID id, uint loc)
c48d2125f1e2 - Moved code for scanning character literals to separate function. aziz parents: 17 diff changeset	189 {
c48d2125f1e2 - Moved code for scanning character literals to separate function. aziz parents: 17 diff changeset	190 this.id = id;
c48d2125f1e2 - Moved code for scanning character literals to separate function. aziz parents: 17 diff changeset	191 this.type = type;
c48d2125f1e2 - Moved code for scanning character literals to separate function. aziz parents: 17 diff changeset	192 this.loc = loc;
c48d2125f1e2 - Moved code for scanning character literals to separate function. aziz parents: 17 diff changeset	193 }
c48d2125f1e2 - Moved code for scanning character literals to separate function. aziz parents: 17 diff changeset	194 }
c48d2125f1e2 - Moved code for scanning character literals to separate function. aziz parents: 17 diff changeset	195
0 8ba2570de175 Initial import. aziz parents: diff changeset	196 class Lexer
8ba2570de175 Initial import. aziz parents: diff changeset	197 {
4 92df59b1ec4a - Started implementation of scan(). aziz parents: 3 diff changeset	198 Token token;
30 426767b94635 - Added code for parsing the '#line' special token. aziz parents: 29 diff changeset	199 string text;
4 92df59b1ec4a - Started implementation of scan(). aziz parents: 3 diff changeset	200 char* p;
92df59b1ec4a - Started implementation of scan(). aziz parents: 3 diff changeset	201 char* end;
92df59b1ec4a - Started implementation of scan(). aziz parents: 3 diff changeset	202
17 9bd0bac79479 - Removed Whitespace from enum list. aziz parents: 16 diff changeset	203 uint loc = 1; /// line of code
9bd0bac79479 - Removed Whitespace from enum list. aziz parents: 16 diff changeset	204
30 426767b94635 - Added code for parsing the '#line' special token. aziz parents: 29 diff changeset	205 char[] fileName;
426767b94635 - Added code for parsing the '#line' special token. aziz parents: 29 diff changeset	206
18 c48d2125f1e2 - Moved code for scanning character literals to separate function. aziz parents: 17 diff changeset	207 Problem[] errors;
c48d2125f1e2 - Moved code for scanning character literals to separate function. aziz parents: 17 diff changeset	208
28 3a9daccf7d96 - Added table for identifiers to Lexer. aziz parents: 27 diff changeset	209 Identifier[string] idtable;
3a9daccf7d96 - Added table for identifiers to Lexer. aziz parents: 27 diff changeset	210
30 426767b94635 - Added code for parsing the '#line' special token. aziz parents: 29 diff changeset	211 this(string text, string fileName)
4 92df59b1ec4a - Started implementation of scan(). aziz parents: 3 diff changeset	212 {
30 426767b94635 - Added code for parsing the '#line' special token. aziz parents: 29 diff changeset	213 this.fileName = fileName;
426767b94635 - Added code for parsing the '#line' special token. aziz parents: 29 diff changeset	214
4 92df59b1ec4a - Started implementation of scan(). aziz parents: 3 diff changeset	215 this.text = text;
39 69b940398d7b - Added unittest to test correct parsing of operator tokens. aziz parents: 38 diff changeset	216 if (text[$-1] != 0)
69b940398d7b - Added unittest to test correct parsing of operator tokens. aziz parents: 38 diff changeset	217 {
69b940398d7b - Added unittest to test correct parsing of operator tokens. aziz parents: 38 diff changeset	218 this.text.length = this.text.length + 1;
69b940398d7b - Added unittest to test correct parsing of operator tokens. aziz parents: 38 diff changeset	219 this.text[$-1] = 0;
69b940398d7b - Added unittest to test correct parsing of operator tokens. aziz parents: 38 diff changeset	220 }
4 92df59b1ec4a - Started implementation of scan(). aziz parents: 3 diff changeset	221
92df59b1ec4a - Started implementation of scan(). aziz parents: 3 diff changeset	222 this.p = this.text.ptr;
92df59b1ec4a - Started implementation of scan(). aziz parents: 3 diff changeset	223 this.end = this.p + this.text.length;
28 3a9daccf7d96 - Added table for identifiers to Lexer. aziz parents: 27 diff changeset	224
3a9daccf7d96 - Added table for identifiers to Lexer. aziz parents: 27 diff changeset	225 loadKeywords();
4 92df59b1ec4a - Started implementation of scan(). aziz parents: 3 diff changeset	226 }
92df59b1ec4a - Started implementation of scan(). aziz parents: 3 diff changeset	227
92df59b1ec4a - Started implementation of scan(). aziz parents: 3 diff changeset	228 public void scan(out Token t)
3 4bbce78bfb1e - Added TOK enum. aziz parents: 2 diff changeset	229 {
4 92df59b1ec4a - Started implementation of scan(). aziz parents: 3 diff changeset	230 assert(p < end);
0 8ba2570de175 Initial import. aziz parents: diff changeset	231
10 3ee65d6e39c9 - Parsing // comments now. aziz parents: 9 diff changeset	232 uint c = *p;
4 92df59b1ec4a - Started implementation of scan(). aziz parents: 3 diff changeset	233
5 79b4e8848794 - Started writing XML generator. aziz parents: 4 diff changeset	234 while(1)
4 92df59b1ec4a - Started implementation of scan(). aziz parents: 3 diff changeset	235 {
5 79b4e8848794 - Started writing XML generator. aziz parents: 4 diff changeset	236 t.start = p;
16 476e8e55c1d4 - Added Whitespace to the character properties table. aziz parents: 15 diff changeset	237
5 79b4e8848794 - Started writing XML generator. aziz parents: 4 diff changeset	238 if (c == 0)
79b4e8848794 - Started writing XML generator. aziz parents: 4 diff changeset	239 {
51 cadd2bfe686c - Displaying error messages in XML. aziz parents: 50 diff changeset	240 assert(*p == 0);
17 9bd0bac79479 - Removed Whitespace from enum list. aziz parents: 16 diff changeset	241 ++p;
51 cadd2bfe686c - Displaying error messages in XML. aziz parents: 50 diff changeset	242 assert(p == end);
5 79b4e8848794 - Started writing XML generator. aziz parents: 4 diff changeset	243 t.type = TOK.EOF;
17 9bd0bac79479 - Removed Whitespace from enum list. aziz parents: 16 diff changeset	244 t.end = p;
5 79b4e8848794 - Started writing XML generator. aziz parents: 4 diff changeset	245 return;
79b4e8848794 - Started writing XML generator. aziz parents: 4 diff changeset	246 }
4 92df59b1ec4a - Started implementation of scan(). aziz parents: 3 diff changeset	247
17 9bd0bac79479 - Removed Whitespace from enum list. aziz parents: 16 diff changeset	248 if (c == '\n')
9bd0bac79479 - Removed Whitespace from enum list. aziz parents: 16 diff changeset	249 {
9bd0bac79479 - Removed Whitespace from enum list. aziz parents: 16 diff changeset	250 c = *++p;
9bd0bac79479 - Removed Whitespace from enum list. aziz parents: 16 diff changeset	251 ++loc;
9bd0bac79479 - Removed Whitespace from enum list. aziz parents: 16 diff changeset	252 continue;
9bd0bac79479 - Removed Whitespace from enum list. aziz parents: 16 diff changeset	253 }
9bd0bac79479 - Removed Whitespace from enum list. aziz parents: 16 diff changeset	254 else if (c == '\r')
9bd0bac79479 - Removed Whitespace from enum list. aziz parents: 16 diff changeset	255 {
9bd0bac79479 - Removed Whitespace from enum list. aziz parents: 16 diff changeset	256 c = *++p;
9bd0bac79479 - Removed Whitespace from enum list. aziz parents: 16 diff changeset	257 if (c != '\n')
9bd0bac79479 - Removed Whitespace from enum list. aziz parents: 16 diff changeset	258 ++loc;
9bd0bac79479 - Removed Whitespace from enum list. aziz parents: 16 diff changeset	259 continue;
9bd0bac79479 - Removed Whitespace from enum list. aziz parents: 16 diff changeset	260 }
32 d7011daa4740 - Added missing commas after the items in the messages table. aziz parents: 31 diff changeset	261 else if (c == LS[0] && p[1] == LS[1] && (p[2] == LS[2] \|\| p[2] == PS[2]))
d7011daa4740 - Added missing commas after the items in the messages table. aziz parents: 31 diff changeset	262 {
d7011daa4740 - Added missing commas after the items in the messages table. aziz parents: 31 diff changeset	263 p += 3;
d7011daa4740 - Added missing commas after the items in the messages table. aziz parents: 31 diff changeset	264 c = *p;
d7011daa4740 - Added missing commas after the items in the messages table. aziz parents: 31 diff changeset	265 continue;
d7011daa4740 - Added missing commas after the items in the messages table. aziz parents: 31 diff changeset	266 }
17 9bd0bac79479 - Removed Whitespace from enum list. aziz parents: 16 diff changeset	267
13 e5211758b63c - Added isidbeg() function. aziz parents: 12 diff changeset	268 if (isidbeg(c))
5 79b4e8848794 - Started writing XML generator. aziz parents: 4 diff changeset	269 {
33 cf3047cf3cd2 - Added code for parsing back quote and raw strings. aziz parents: 32 diff changeset	270 if (c == 'r' && p[1] == '"' && ++p)
31 94f09f4e988e - Added struct for strings to Token with 'pf' = postfix. aziz parents: 30 diff changeset	271 return scanRawStringLiteral(t);
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix. aziz parents: 30 diff changeset	272 if (c == 'x' && p[1] == '"')
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix. aziz parents: 30 diff changeset	273 return scanHexStringLiteral(t);
12 0989206cf73c - Added code to decode Unicode characters in identifiers. aziz parents: 11 diff changeset	274 Lidentifier:
5 79b4e8848794 - Started writing XML generator. aziz parents: 4 diff changeset	275 do
79b4e8848794 - Started writing XML generator. aziz parents: 4 diff changeset	276 { c = *++p; }
49 7f0fa15dcffc - Renamed function. aziz parents: 48 diff changeset	277 while (isident(c) \|\| c & 128 && isUniAlpha(decodeUTF8()))
28 3a9daccf7d96 - Added table for identifiers to Lexer. aziz parents: 27 diff changeset	278
5 79b4e8848794 - Started writing XML generator. aziz parents: 4 diff changeset	279 t.end = p;
28 3a9daccf7d96 - Added table for identifiers to Lexer. aziz parents: 27 diff changeset	280
3a9daccf7d96 - Added table for identifiers to Lexer. aziz parents: 27 diff changeset	281 string str = t.span;
3a9daccf7d96 - Added table for identifiers to Lexer. aziz parents: 27 diff changeset	282 Identifier* id = str in idtable;
3a9daccf7d96 - Added table for identifiers to Lexer. aziz parents: 27 diff changeset	283
3a9daccf7d96 - Added table for identifiers to Lexer. aziz parents: 27 diff changeset	284 if (!id)
3a9daccf7d96 - Added table for identifiers to Lexer. aziz parents: 27 diff changeset	285 {
3a9daccf7d96 - Added table for identifiers to Lexer. aziz parents: 27 diff changeset	286 idtable[str] = Identifier.Identifier(TOK.Identifier, str);
3a9daccf7d96 - Added table for identifiers to Lexer. aziz parents: 27 diff changeset	287 id = str in idtable;
3a9daccf7d96 - Added table for identifiers to Lexer. aziz parents: 27 diff changeset	288 }
3a9daccf7d96 - Added table for identifiers to Lexer. aziz parents: 27 diff changeset	289 assert(id);
3a9daccf7d96 - Added table for identifiers to Lexer. aziz parents: 27 diff changeset	290 t.type = id.type;
5 79b4e8848794 - Started writing XML generator. aziz parents: 4 diff changeset	291 return;
79b4e8848794 - Started writing XML generator. aziz parents: 4 diff changeset	292 }
79b4e8848794 - Started writing XML generator. aziz parents: 4 diff changeset	293
15 c70c028e47dd - Started implementation of lexing numbers. aziz parents: 14 diff changeset	294 if (isdigit(c))
c70c028e47dd - Started implementation of lexing numbers. aziz parents: 14 diff changeset	295 return scanNumber(t);
c70c028e47dd - Started implementation of lexing numbers. aziz parents: 14 diff changeset	296
8 d4ba94a5a282 - Parsing /* / comments now. aziz* parents: 7 diff changeset	297 if (c == '/')
5 79b4e8848794 - Started writing XML generator. aziz parents: 4 diff changeset	298 {
8 d4ba94a5a282 - Parsing /* / comments now. aziz* parents: 7 diff changeset	299 c = *++p;
14 cdf788d8bdaf - Parsing /= now. aziz parents: 13 diff changeset	300 switch(c)
5 79b4e8848794 - Started writing XML generator. aziz parents: 4 diff changeset	301 {
18 c48d2125f1e2 - Moved code for scanning character literals to separate function. aziz parents: 17 diff changeset	302 case '=':
14 cdf788d8bdaf - Parsing /= now. aziz parents: 13 diff changeset	303 ++p;
32 d7011daa4740 - Added missing commas after the items in the messages table. aziz parents: 31 diff changeset	304 t.type = TOK.DivAssign;
14 cdf788d8bdaf - Parsing /= now. aziz parents: 13 diff changeset	305 t.end = p;
cdf788d8bdaf - Parsing /= now. aziz parents: 13 diff changeset	306 return;
18 c48d2125f1e2 - Moved code for scanning character literals to separate function. aziz parents: 17 diff changeset	307 case '+':
8 d4ba94a5a282 - Parsing /* / comments now. aziz* parents: 7 diff changeset	308 uint level = 1;
32 d7011daa4740 - Added missing commas after the items in the messages table. aziz parents: 31 diff changeset	309 while (1)
d7011daa4740 - Added missing commas after the items in the messages table. aziz parents: 31 diff changeset	310 {
d7011daa4740 - Added missing commas after the items in the messages table. aziz parents: 31 diff changeset	311 c = *++p;
42 c6a9974a6e3c - Optimized scanner of nested comments. aziz parents: 41 diff changeset	312 LswitchNC: // only jumped to from default case of next switch(c)
32 d7011daa4740 - Added missing commas after the items in the messages table. aziz parents: 31 diff changeset	313 switch (c)
d7011daa4740 - Added missing commas after the items in the messages table. aziz parents: 31 diff changeset	314 {
d7011daa4740 - Added missing commas after the items in the messages table. aziz parents: 31 diff changeset	315 case '\r':
d7011daa4740 - Added missing commas after the items in the messages table. aziz parents: 31 diff changeset	316 if (p[1] == '\n')
d7011daa4740 - Added missing commas after the items in the messages table. aziz parents: 31 diff changeset	317 ++p;
d7011daa4740 - Added missing commas after the items in the messages table. aziz parents: 31 diff changeset	318 case '\n':
d7011daa4740 - Added missing commas after the items in the messages table. aziz parents: 31 diff changeset	319 ++loc;
d7011daa4740 - Added missing commas after the items in the messages table. aziz parents: 31 diff changeset	320 continue;
d7011daa4740 - Added missing commas after the items in the messages table. aziz parents: 31 diff changeset	321 case 0, _Z_:
d7011daa4740 - Added missing commas after the items in the messages table. aziz parents: 31 diff changeset	322 error(MID.UnterminatedNestedComment);
d7011daa4740 - Added missing commas after the items in the messages table. aziz parents: 31 diff changeset	323 goto LreturnNC;
42 c6a9974a6e3c - Optimized scanner of nested comments. aziz parents: 41 diff changeset	324 default:
c6a9974a6e3c - Optimized scanner of nested comments. aziz parents: 41 diff changeset	325 }
c6a9974a6e3c - Optimized scanner of nested comments. aziz parents: 41 diff changeset	326
c6a9974a6e3c - Optimized scanner of nested comments. aziz parents: 41 diff changeset	327 c <<= 8;
c6a9974a6e3c - Optimized scanner of nested comments. aziz parents: 41 diff changeset	328 c \|= *++p;
c6a9974a6e3c - Optimized scanner of nested comments. aziz parents: 41 diff changeset	329 switch (c)
c6a9974a6e3c - Optimized scanner of nested comments. aziz parents: 41 diff changeset	330 {
c6a9974a6e3c - Optimized scanner of nested comments. aziz parents: 41 diff changeset	331 case 0x2F2B: // /+
c6a9974a6e3c - Optimized scanner of nested comments. aziz parents: 41 diff changeset	332 ++level;
c6a9974a6e3c - Optimized scanner of nested comments. aziz parents: 41 diff changeset	333 continue;
c6a9974a6e3c - Optimized scanner of nested comments. aziz parents: 41 diff changeset	334 case 0x2B2F: // +/
c6a9974a6e3c - Optimized scanner of nested comments. aziz parents: 41 diff changeset	335 if (--level == 0)
c6a9974a6e3c - Optimized scanner of nested comments. aziz parents: 41 diff changeset	336 {
c6a9974a6e3c - Optimized scanner of nested comments. aziz parents: 41 diff changeset	337 ++p;
c6a9974a6e3c - Optimized scanner of nested comments. aziz parents: 41 diff changeset	338 LreturnNC:
c6a9974a6e3c - Optimized scanner of nested comments. aziz parents: 41 diff changeset	339 t.type = TOK.Comment;
c6a9974a6e3c - Optimized scanner of nested comments. aziz parents: 41 diff changeset	340 t.end = p;
c6a9974a6e3c - Optimized scanner of nested comments. aziz parents: 41 diff changeset	341 return;
32 d7011daa4740 - Added missing commas after the items in the messages table. aziz parents: 31 diff changeset	342 }
d7011daa4740 - Added missing commas after the items in the messages table. aziz parents: 31 diff changeset	343 continue;
42 c6a9974a6e3c - Optimized scanner of nested comments. aziz parents: 41 diff changeset	344 case 0xE280: // LS[0..1] \|\| PS[0..1]
c6a9974a6e3c - Optimized scanner of nested comments. aziz parents: 41 diff changeset	345 if (p[1] == LS[2] \|\| p[1] == PS[2])
c6a9974a6e3c - Optimized scanner of nested comments. aziz parents: 41 diff changeset	346 {
c6a9974a6e3c - Optimized scanner of nested comments. aziz parents: 41 diff changeset	347 ++loc;
c6a9974a6e3c - Optimized scanner of nested comments. aziz parents: 41 diff changeset	348 ++p;
c6a9974a6e3c - Optimized scanner of nested comments. aziz parents: 41 diff changeset	349 }
43 1845c23dd056 - Matched some parts of the scanner of block comments to the scanner of nested comments. aziz parents: 42 diff changeset	350 continue;
32 d7011daa4740 - Added missing commas after the items in the messages table. aziz parents: 31 diff changeset	351 default:
42 c6a9974a6e3c - Optimized scanner of nested comments. aziz parents: 41 diff changeset	352 c &= char.max;
c6a9974a6e3c - Optimized scanner of nested comments. aziz parents: 41 diff changeset	353 goto LswitchNC;
32 d7011daa4740 - Added missing commas after the items in the messages table. aziz parents: 31 diff changeset	354 }
d7011daa4740 - Added missing commas after the items in the messages table. aziz parents: 31 diff changeset	355 }
d7011daa4740 - Added missing commas after the items in the messages table. aziz parents: 31 diff changeset	356 case '*':
d7011daa4740 - Added missing commas after the items in the messages table. aziz parents: 31 diff changeset	357 while (1)
7 07e45c06a024 - Parsing nested comments correctly now. aziz parents: 5 diff changeset	358 {
43 1845c23dd056 - Matched some parts of the scanner of block comments to the scanner of nested comments. aziz parents: 42 diff changeset	359 c = *++p;
1845c23dd056 - Matched some parts of the scanner of block comments to the scanner of nested comments. aziz parents: 42 diff changeset	360 LswitchBC: // only jumped to from default case of next switch(c)
32 d7011daa4740 - Added missing commas after the items in the messages table. aziz parents: 31 diff changeset	361 switch (c)
8 d4ba94a5a282 - Parsing /* / comments now. aziz* parents: 7 diff changeset	362 {
32 d7011daa4740 - Added missing commas after the items in the messages table. aziz parents: 31 diff changeset	363 case '\r':
d7011daa4740 - Added missing commas after the items in the messages table. aziz parents: 31 diff changeset	364 if (p[1] == '\n')
d7011daa4740 - Added missing commas after the items in the messages table. aziz parents: 31 diff changeset	365 ++p;
d7011daa4740 - Added missing commas after the items in the messages table. aziz parents: 31 diff changeset	366 case '\n':
d7011daa4740 - Added missing commas after the items in the messages table. aziz parents: 31 diff changeset	367 ++loc;
d7011daa4740 - Added missing commas after the items in the messages table. aziz parents: 31 diff changeset	368 continue;
d7011daa4740 - Added missing commas after the items in the messages table. aziz parents: 31 diff changeset	369 case 0, _Z_:
d7011daa4740 - Added missing commas after the items in the messages table. aziz parents: 31 diff changeset	370 error(MID.UnterminatedBlockComment);
d7011daa4740 - Added missing commas after the items in the messages table. aziz parents: 31 diff changeset	371 goto LreturnBC;
d7011daa4740 - Added missing commas after the items in the messages table. aziz parents: 31 diff changeset	372 default:
8 d4ba94a5a282 - Parsing /* / comments now. aziz* parents: 7 diff changeset	373 }
41 2b7be1d67d4d - Optimized scanner of block comments. aziz parents: 40 diff changeset	374
2b7be1d67d4d - Optimized scanner of block comments. aziz parents: 40 diff changeset	375 c <<= 8;
2b7be1d67d4d - Optimized scanner of block comments. aziz parents: 40 diff changeset	376 c \|= *++p;
2b7be1d67d4d - Optimized scanner of block comments. aziz parents: 40 diff changeset	377 switch (c)
2b7be1d67d4d - Optimized scanner of block comments. aziz parents: 40 diff changeset	378 {
2b7be1d67d4d - Optimized scanner of block comments. aziz parents: 40 diff changeset	379 case 0x2A2F: // */
2b7be1d67d4d - Optimized scanner of block comments. aziz parents: 40 diff changeset	380 ++p;
2b7be1d67d4d - Optimized scanner of block comments. aziz parents: 40 diff changeset	381 LreturnBC:
2b7be1d67d4d - Optimized scanner of block comments. aziz parents: 40 diff changeset	382 t.type = TOK.Comment;
2b7be1d67d4d - Optimized scanner of block comments. aziz parents: 40 diff changeset	383 t.end = p;
2b7be1d67d4d - Optimized scanner of block comments. aziz parents: 40 diff changeset	384 return;
2b7be1d67d4d - Optimized scanner of block comments. aziz parents: 40 diff changeset	385 case 0xE280: // LS[0..1] \|\| PS[0..1]
2b7be1d67d4d - Optimized scanner of block comments. aziz parents: 40 diff changeset	386 if (p[1] == LS[2] \|\| p[1] == PS[2])
2b7be1d67d4d - Optimized scanner of block comments. aziz parents: 40 diff changeset	387 {
2b7be1d67d4d - Optimized scanner of block comments. aziz parents: 40 diff changeset	388 ++loc;
43 1845c23dd056 - Matched some parts of the scanner of block comments to the scanner of nested comments. aziz parents: 42 diff changeset	389 ++p;
41 2b7be1d67d4d - Optimized scanner of block comments. aziz parents: 40 diff changeset	390 }
43 1845c23dd056 - Matched some parts of the scanner of block comments to the scanner of nested comments. aziz parents: 42 diff changeset	391 continue;
41 2b7be1d67d4d - Optimized scanner of block comments. aziz parents: 40 diff changeset	392 default:
2b7be1d67d4d - Optimized scanner of block comments. aziz parents: 40 diff changeset	393 c &= char.max;
43 1845c23dd056 - Matched some parts of the scanner of block comments to the scanner of nested comments. aziz parents: 42 diff changeset	394 goto LswitchBC;
41 2b7be1d67d4d - Optimized scanner of block comments. aziz parents: 40 diff changeset	395 }
32 d7011daa4740 - Added missing commas after the items in the messages table. aziz parents: 31 diff changeset	396 }
d7011daa4740 - Added missing commas after the items in the messages table. aziz parents: 31 diff changeset	397 assert(0);
d7011daa4740 - Added missing commas after the items in the messages table. aziz parents: 31 diff changeset	398 case '/':
d7011daa4740 - Added missing commas after the items in the messages table. aziz parents: 31 diff changeset	399 while (1)
7 07e45c06a024 - Parsing nested comments correctly now. aziz parents: 5 diff changeset	400 {
8 d4ba94a5a282 - Parsing /* / comments now. aziz* parents: 7 diff changeset	401 c = *++p;
32 d7011daa4740 - Added missing commas after the items in the messages table. aziz parents: 31 diff changeset	402 switch (c)
d7011daa4740 - Added missing commas after the items in the messages table. aziz parents: 31 diff changeset	403 {
d7011daa4740 - Added missing commas after the items in the messages table. aziz parents: 31 diff changeset	404 case '\r':
d7011daa4740 - Added missing commas after the items in the messages table. aziz parents: 31 diff changeset	405 if (p[1] == '\n')
d7011daa4740 - Added missing commas after the items in the messages table. aziz parents: 31 diff changeset	406 ++p;
d7011daa4740 - Added missing commas after the items in the messages table. aziz parents: 31 diff changeset	407 case '\n':
d7011daa4740 - Added missing commas after the items in the messages table. aziz parents: 31 diff changeset	408 case 0, _Z_:
11 dffcdaa7c47a - Added Unicode line and paragraph separators. aziz parents: 10 diff changeset	409 break;
32 d7011daa4740 - Added missing commas after the items in the messages table. aziz parents: 31 diff changeset	410 case LS[0]:
d7011daa4740 - Added missing commas after the items in the messages table. aziz parents: 31 diff changeset	411 if (p[1] == LS[1] && (p[2] == LS[2] \|\| p[2] == PS[2]))
d7011daa4740 - Added missing commas after the items in the messages table. aziz parents: 31 diff changeset	412 break;
d7011daa4740 - Added missing commas after the items in the messages table. aziz parents: 31 diff changeset	413 continue;
d7011daa4740 - Added missing commas after the items in the messages table. aziz parents: 31 diff changeset	414 default:
d7011daa4740 - Added missing commas after the items in the messages table. aziz parents: 31 diff changeset	415 continue;
d7011daa4740 - Added missing commas after the items in the messages table. aziz parents: 31 diff changeset	416 }
d7011daa4740 - Added missing commas after the items in the messages table. aziz parents: 31 diff changeset	417 t.type = TOK.Comment;
d7011daa4740 - Added missing commas after the items in the messages table. aziz parents: 31 diff changeset	418 t.end = p;
d7011daa4740 - Added missing commas after the items in the messages table. aziz parents: 31 diff changeset	419 return;
d7011daa4740 - Added missing commas after the items in the messages table. aziz parents: 31 diff changeset	420 }
d7011daa4740 - Added missing commas after the items in the messages table. aziz parents: 31 diff changeset	421 default:
d7011daa4740 - Added missing commas after the items in the messages table. aziz parents: 31 diff changeset	422 t.type = TOK.Div;
10 3ee65d6e39c9 - Parsing // comments now. aziz parents: 9 diff changeset	423 t.end = p;
3ee65d6e39c9 - Parsing // comments now. aziz parents: 9 diff changeset	424 return;
3ee65d6e39c9 - Parsing // comments now. aziz parents: 9 diff changeset	425 }
5 79b4e8848794 - Started writing XML generator. aziz parents: 4 diff changeset	426 }
79b4e8848794 - Started writing XML generator. aziz parents: 4 diff changeset	427
36 3c7210a722f7 - Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens. aziz parents: 35 diff changeset	428 switch (c)
20 d6adfbd7c513 - Added code for parsing braces. aziz parents: 19 diff changeset	429 {
48 c2e0e0269c28 - Added code for scanning escape string literals. aziz parents: 47 diff changeset	430 case '\'':
c2e0e0269c28 - Added code for scanning escape string literals. aziz parents: 47 diff changeset	431 return scanCharacterLiteral(t);
c2e0e0269c28 - Added code for scanning escape string literals. aziz parents: 47 diff changeset	432 case '`':
c2e0e0269c28 - Added code for scanning escape string literals. aziz parents: 47 diff changeset	433 return scanRawStringLiteral(t);
c2e0e0269c28 - Added code for scanning escape string literals. aziz parents: 47 diff changeset	434 case '"':
c2e0e0269c28 - Added code for scanning escape string literals. aziz parents: 47 diff changeset	435 return scanNormalStringLiteral(t);
c2e0e0269c28 - Added code for scanning escape string literals. aziz parents: 47 diff changeset	436 case '\\':
c2e0e0269c28 - Added code for scanning escape string literals. aziz parents: 47 diff changeset	437 char[] buffer;
c2e0e0269c28 - Added code for scanning escape string literals. aziz parents: 47 diff changeset	438 do
c2e0e0269c28 - Added code for scanning escape string literals. aziz parents: 47 diff changeset	439 {
c2e0e0269c28 - Added code for scanning escape string literals. aziz parents: 47 diff changeset	440 ++p;
c2e0e0269c28 - Added code for scanning escape string literals. aziz parents: 47 diff changeset	441 c = scanEscapeSequence();
50 4a27b7840ea9 - Return error code 0xFFFF from scanEscapeSequence(). aziz parents: 49 diff changeset	442 if (c == 0xFFFF)
4a27b7840ea9 - Return error code 0xFFFF from scanEscapeSequence(). aziz parents: 49 diff changeset	443 break;
48 c2e0e0269c28 - Added code for scanning escape string literals. aziz parents: 47 diff changeset	444 if (c < 128)
c2e0e0269c28 - Added code for scanning escape string literals. aziz parents: 47 diff changeset	445 buffer ~= c;
c2e0e0269c28 - Added code for scanning escape string literals. aziz parents: 47 diff changeset	446 else
c2e0e0269c28 - Added code for scanning escape string literals. aziz parents: 47 diff changeset	447 encodeUTF8(buffer, c);
c2e0e0269c28 - Added code for scanning escape string literals. aziz parents: 47 diff changeset	448 } while (*p == '\\')
c2e0e0269c28 - Added code for scanning escape string literals. aziz parents: 47 diff changeset	449 buffer ~= 0;
c2e0e0269c28 - Added code for scanning escape string literals. aziz parents: 47 diff changeset	450 t.type = TOK.String;
c2e0e0269c28 - Added code for scanning escape string literals. aziz parents: 47 diff changeset	451 t.str = buffer;
c2e0e0269c28 - Added code for scanning escape string literals. aziz parents: 47 diff changeset	452 t.end = p;
c2e0e0269c28 - Added code for scanning escape string literals. aziz parents: 47 diff changeset	453 return;
38 640c45aaaaee - Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens. aziz parents: 37 diff changeset	454 case '>': /* > >= >> >>= >>> >>>= */
640c45aaaaee - Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens. aziz parents: 37 diff changeset	455 c = *++p;
640c45aaaaee - Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens. aziz parents: 37 diff changeset	456 switch (c)
640c45aaaaee - Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens. aziz parents: 37 diff changeset	457 {
640c45aaaaee - Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens. aziz parents: 37 diff changeset	458 case '=':
640c45aaaaee - Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens. aziz parents: 37 diff changeset	459 t.type = TOK.GreaterEqual;
640c45aaaaee - Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens. aziz parents: 37 diff changeset	460 goto Lcommon;
640c45aaaaee - Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens. aziz parents: 37 diff changeset	461 case '>':
640c45aaaaee - Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens. aziz parents: 37 diff changeset	462 if (p[1] == '>')
640c45aaaaee - Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens. aziz parents: 37 diff changeset	463 {
640c45aaaaee - Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens. aziz parents: 37 diff changeset	464 ++p;
640c45aaaaee - Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens. aziz parents: 37 diff changeset	465 if (p[1] == '=')
640c45aaaaee - Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens. aziz parents: 37 diff changeset	466 { ++p;
640c45aaaaee - Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens. aziz parents: 37 diff changeset	467 t.type = TOK.URShiftAssign;
640c45aaaaee - Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens. aziz parents: 37 diff changeset	468 }
640c45aaaaee - Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens. aziz parents: 37 diff changeset	469 else
640c45aaaaee - Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens. aziz parents: 37 diff changeset	470 t.type = TOK.URShift;
640c45aaaaee - Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens. aziz parents: 37 diff changeset	471 }
640c45aaaaee - Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens. aziz parents: 37 diff changeset	472 else if (p[1] == '=')
640c45aaaaee - Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens. aziz parents: 37 diff changeset	473 {
640c45aaaaee - Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens. aziz parents: 37 diff changeset	474 ++p;
640c45aaaaee - Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens. aziz parents: 37 diff changeset	475 t.type = TOK.RShiftAssign;
640c45aaaaee - Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens. aziz parents: 37 diff changeset	476 }
640c45aaaaee - Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens. aziz parents: 37 diff changeset	477 else
640c45aaaaee - Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens. aziz parents: 37 diff changeset	478 t.type = TOK.RShift;
640c45aaaaee - Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens. aziz parents: 37 diff changeset	479 goto Lcommon;
640c45aaaaee - Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens. aziz parents: 37 diff changeset	480 default:
640c45aaaaee - Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens. aziz parents: 37 diff changeset	481 t.type = TOK.Greater;
640c45aaaaee - Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens. aziz parents: 37 diff changeset	482 goto Lcommon2;
640c45aaaaee - Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens. aziz parents: 37 diff changeset	483 }
640c45aaaaee - Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens. aziz parents: 37 diff changeset	484 assert(0);
36 3c7210a722f7 - Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens. aziz parents: 35 diff changeset	485 case '<': /* < <= <> <>= << <<= */
3c7210a722f7 - Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens. aziz parents: 35 diff changeset	486 c = *++p;
3c7210a722f7 - Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens. aziz parents: 35 diff changeset	487 switch (c)
3c7210a722f7 - Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens. aziz parents: 35 diff changeset	488 {
3c7210a722f7 - Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens. aziz parents: 35 diff changeset	489 case '=':
3c7210a722f7 - Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens. aziz parents: 35 diff changeset	490 t.type = TOK.LessEqual;
3c7210a722f7 - Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens. aziz parents: 35 diff changeset	491 goto Lcommon;
3c7210a722f7 - Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens. aziz parents: 35 diff changeset	492 case '<':
3c7210a722f7 - Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens. aziz parents: 35 diff changeset	493 if (p[1] == '=') {
3c7210a722f7 - Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens. aziz parents: 35 diff changeset	494 ++p;
3c7210a722f7 - Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens. aziz parents: 35 diff changeset	495 t.type = TOK.LShiftAssign;
3c7210a722f7 - Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens. aziz parents: 35 diff changeset	496 }
3c7210a722f7 - Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens. aziz parents: 35 diff changeset	497 else
3c7210a722f7 - Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens. aziz parents: 35 diff changeset	498 t.type = TOK.LShift;
3c7210a722f7 - Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens. aziz parents: 35 diff changeset	499 goto Lcommon;
3c7210a722f7 - Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens. aziz parents: 35 diff changeset	500 case '>':
3c7210a722f7 - Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens. aziz parents: 35 diff changeset	501 if (p[1] == '=') {
3c7210a722f7 - Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens. aziz parents: 35 diff changeset	502 ++p;
3c7210a722f7 - Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens. aziz parents: 35 diff changeset	503 t.type = TOK.LorEorG;
3c7210a722f7 - Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens. aziz parents: 35 diff changeset	504 }
3c7210a722f7 - Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens. aziz parents: 35 diff changeset	505 else
3c7210a722f7 - Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens. aziz parents: 35 diff changeset	506 t.type = TOK.LorG;
3c7210a722f7 - Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens. aziz parents: 35 diff changeset	507 goto Lcommon;
3c7210a722f7 - Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens. aziz parents: 35 diff changeset	508 default:
38 640c45aaaaee - Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens. aziz parents: 37 diff changeset	509 t.type = TOK.Less;
36 3c7210a722f7 - Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens. aziz parents: 35 diff changeset	510 goto Lcommon2;
3c7210a722f7 - Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens. aziz parents: 35 diff changeset	511 }
3c7210a722f7 - Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens. aziz parents: 35 diff changeset	512 assert(0);
37 7f3bcb97d017 - Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens. aziz parents: 36 diff changeset	513 case '!': /* ! !< !> !<= !>= !<> !<>= */
35 c470b9356e35 - Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens. aziz parents: 34 diff changeset	514 c = *++p;
c470b9356e35 - Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens. aziz parents: 34 diff changeset	515 switch (c)
c470b9356e35 - Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens. aziz parents: 34 diff changeset	516 {
c470b9356e35 - Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens. aziz parents: 34 diff changeset	517 case '<':
c470b9356e35 - Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens. aziz parents: 34 diff changeset	518 c = *++p;
c470b9356e35 - Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens. aziz parents: 34 diff changeset	519 if (c == '>')
c470b9356e35 - Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens. aziz parents: 34 diff changeset	520 {
c470b9356e35 - Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens. aziz parents: 34 diff changeset	521 if (p[1] == '=') {
c470b9356e35 - Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens. aziz parents: 34 diff changeset	522 ++p;
c470b9356e35 - Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens. aziz parents: 34 diff changeset	523 t.type = TOK.Unordered;
c470b9356e35 - Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens. aziz parents: 34 diff changeset	524 }
c470b9356e35 - Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens. aziz parents: 34 diff changeset	525 else
c470b9356e35 - Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens. aziz parents: 34 diff changeset	526 t.type = TOK.UorE;
c470b9356e35 - Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens. aziz parents: 34 diff changeset	527 }
c470b9356e35 - Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens. aziz parents: 34 diff changeset	528 else if (c == '=')
c470b9356e35 - Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens. aziz parents: 34 diff changeset	529 {
c470b9356e35 - Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens. aziz parents: 34 diff changeset	530 t.type = TOK.UorG;
c470b9356e35 - Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens. aziz parents: 34 diff changeset	531 }
c470b9356e35 - Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens. aziz parents: 34 diff changeset	532 else {
c470b9356e35 - Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens. aziz parents: 34 diff changeset	533 t.type = TOK.UorGorE;
c470b9356e35 - Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens. aziz parents: 34 diff changeset	534 goto Lcommon2;
c470b9356e35 - Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens. aziz parents: 34 diff changeset	535 }
c470b9356e35 - Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens. aziz parents: 34 diff changeset	536 goto Lcommon;
c470b9356e35 - Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens. aziz parents: 34 diff changeset	537 case '>':
c470b9356e35 - Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens. aziz parents: 34 diff changeset	538 if (p[1] == '=')
c470b9356e35 - Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens. aziz parents: 34 diff changeset	539 {
c470b9356e35 - Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens. aziz parents: 34 diff changeset	540 ++p;
c470b9356e35 - Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens. aziz parents: 34 diff changeset	541 t.type = TOK.UorL;
c470b9356e35 - Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens. aziz parents: 34 diff changeset	542 }
c470b9356e35 - Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens. aziz parents: 34 diff changeset	543 else
c470b9356e35 - Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens. aziz parents: 34 diff changeset	544 t.type = TOK.UorLorE;
c470b9356e35 - Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens. aziz parents: 34 diff changeset	545 goto Lcommon;
c470b9356e35 - Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens. aziz parents: 34 diff changeset	546 case '=':
c470b9356e35 - Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens. aziz parents: 34 diff changeset	547 t.type = TOK.NotEqual;
c470b9356e35 - Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens. aziz parents: 34 diff changeset	548 goto Lcommon;
c470b9356e35 - Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens. aziz parents: 34 diff changeset	549 default:
c470b9356e35 - Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens. aziz parents: 34 diff changeset	550 t.type = TOK.Not;
c470b9356e35 - Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens. aziz parents: 34 diff changeset	551 goto Lcommon2;
c470b9356e35 - Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens. aziz parents: 34 diff changeset	552 }
c470b9356e35 - Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens. aziz parents: 34 diff changeset	553 assert(0);
36 3c7210a722f7 - Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens. aziz parents: 35 diff changeset	554 case '.': /* . .. ... */
22 b05fff8e2ce4 - Added code for parsing Dot, Slice and Ellipses tokens. aziz parents: 21 diff changeset	555 if (p[1] == '.')
b05fff8e2ce4 - Added code for parsing Dot, Slice and Ellipses tokens. aziz parents: 21 diff changeset	556 {
b05fff8e2ce4 - Added code for parsing Dot, Slice and Ellipses tokens. aziz parents: 21 diff changeset	557 ++p;
b05fff8e2ce4 - Added code for parsing Dot, Slice and Ellipses tokens. aziz parents: 21 diff changeset	558 if (p[1] == '.') {
b05fff8e2ce4 - Added code for parsing Dot, Slice and Ellipses tokens. aziz parents: 21 diff changeset	559 ++p;
b05fff8e2ce4 - Added code for parsing Dot, Slice and Ellipses tokens. aziz parents: 21 diff changeset	560 t.type = TOK.Ellipses;
b05fff8e2ce4 - Added code for parsing Dot, Slice and Ellipses tokens. aziz parents: 21 diff changeset	561 }
b05fff8e2ce4 - Added code for parsing Dot, Slice and Ellipses tokens. aziz parents: 21 diff changeset	562 else
b05fff8e2ce4 - Added code for parsing Dot, Slice and Ellipses tokens. aziz parents: 21 diff changeset	563 t.type = TOK.Slice;
b05fff8e2ce4 - Added code for parsing Dot, Slice and Ellipses tokens. aziz parents: 21 diff changeset	564 }
b05fff8e2ce4 - Added code for parsing Dot, Slice and Ellipses tokens. aziz parents: 21 diff changeset	565 else
b05fff8e2ce4 - Added code for parsing Dot, Slice and Ellipses tokens. aziz parents: 21 diff changeset	566 t.type = TOK.Dot;
b05fff8e2ce4 - Added code for parsing Dot, Slice and Ellipses tokens. aziz parents: 21 diff changeset	567 goto Lcommon;
36 3c7210a722f7 - Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens. aziz parents: 35 diff changeset	568 case '\|': /* \| \|\| \|= */
23 1a7903701a3d - Added code for parsing OrAssign, OrLogical and OrBinary tokens. aziz parents: 22 diff changeset	569 c = *++p;
1a7903701a3d - Added code for parsing OrAssign, OrLogical and OrBinary tokens. aziz parents: 22 diff changeset	570 if (c == '=')
1a7903701a3d - Added code for parsing OrAssign, OrLogical and OrBinary tokens. aziz parents: 22 diff changeset	571 t.type = TOK.OrAssign;
1a7903701a3d - Added code for parsing OrAssign, OrLogical and OrBinary tokens. aziz parents: 22 diff changeset	572 else if (c == '\|')
1a7903701a3d - Added code for parsing OrAssign, OrLogical and OrBinary tokens. aziz parents: 22 diff changeset	573 t.type = TOK.OrLogical;
26 c3d7373db241 - Added code for parsing Assign and Equal tokens. aziz parents: 25 diff changeset	574 else {
23 1a7903701a3d - Added code for parsing OrAssign, OrLogical and OrBinary tokens. aziz parents: 22 diff changeset	575 t.type = TOK.OrBinary;
26 c3d7373db241 - Added code for parsing Assign and Equal tokens. aziz parents: 25 diff changeset	576 goto Lcommon2;
c3d7373db241 - Added code for parsing Assign and Equal tokens. aziz parents: 25 diff changeset	577 }
23 1a7903701a3d - Added code for parsing OrAssign, OrLogical and OrBinary tokens. aziz parents: 22 diff changeset	578 goto Lcommon;
36 3c7210a722f7 - Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens. aziz parents: 35 diff changeset	579 case '&': /* & && &= */
24 903f91163f23 - Added code for parsing AndAssign, AndLogical and AndBinary tokens. aziz parents: 23 diff changeset	580 c = *++p;
903f91163f23 - Added code for parsing AndAssign, AndLogical and AndBinary tokens. aziz parents: 23 diff changeset	581 if (c == '=')
903f91163f23 - Added code for parsing AndAssign, AndLogical and AndBinary tokens. aziz parents: 23 diff changeset	582 t.type = TOK.AndAssign;
903f91163f23 - Added code for parsing AndAssign, AndLogical and AndBinary tokens. aziz parents: 23 diff changeset	583 else if (c == '&')
903f91163f23 - Added code for parsing AndAssign, AndLogical and AndBinary tokens. aziz parents: 23 diff changeset	584 t.type = TOK.AndLogical;
26 c3d7373db241 - Added code for parsing Assign and Equal tokens. aziz parents: 25 diff changeset	585 else {
24 903f91163f23 - Added code for parsing AndAssign, AndLogical and AndBinary tokens. aziz parents: 23 diff changeset	586 t.type = TOK.AndBinary;
26 c3d7373db241 - Added code for parsing Assign and Equal tokens. aziz parents: 25 diff changeset	587 goto Lcommon2;
c3d7373db241 - Added code for parsing Assign and Equal tokens. aziz parents: 25 diff changeset	588 }
24 903f91163f23 - Added code for parsing AndAssign, AndLogical and AndBinary tokens. aziz parents: 23 diff changeset	589 goto Lcommon;
36 3c7210a722f7 - Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens. aziz parents: 35 diff changeset	590 case '+': /* + ++ += */
25 9c866208b3f6 - Added code for parsing PlusAssign, PlusPlus, Plus, MinusAssign, MinusMinus, Minus tokens. aziz parents: 24 diff changeset	591 c = *++p;
9c866208b3f6 - Added code for parsing PlusAssign, PlusPlus, Plus, MinusAssign, MinusMinus, Minus tokens. aziz parents: 24 diff changeset	592 if (c == '=')
9c866208b3f6 - Added code for parsing PlusAssign, PlusPlus, Plus, MinusAssign, MinusMinus, Minus tokens. aziz parents: 24 diff changeset	593 t.type = TOK.PlusAssign;
9c866208b3f6 - Added code for parsing PlusAssign, PlusPlus, Plus, MinusAssign, MinusMinus, Minus tokens. aziz parents: 24 diff changeset	594 else if (c == '+')
9c866208b3f6 - Added code for parsing PlusAssign, PlusPlus, Plus, MinusAssign, MinusMinus, Minus tokens. aziz parents: 24 diff changeset	595 t.type = TOK.PlusPlus;
26 c3d7373db241 - Added code for parsing Assign and Equal tokens. aziz parents: 25 diff changeset	596 else {
25 9c866208b3f6 - Added code for parsing PlusAssign, PlusPlus, Plus, MinusAssign, MinusMinus, Minus tokens. aziz parents: 24 diff changeset	597 t.type = TOK.Plus;
26 c3d7373db241 - Added code for parsing Assign and Equal tokens. aziz parents: 25 diff changeset	598 goto Lcommon2;
c3d7373db241 - Added code for parsing Assign and Equal tokens. aziz parents: 25 diff changeset	599 }
25 9c866208b3f6 - Added code for parsing PlusAssign, PlusPlus, Plus, MinusAssign, MinusMinus, Minus tokens. aziz parents: 24 diff changeset	600 goto Lcommon;
36 3c7210a722f7 - Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens. aziz parents: 35 diff changeset	601 case '-': /* - -- -= */
25 9c866208b3f6 - Added code for parsing PlusAssign, PlusPlus, Plus, MinusAssign, MinusMinus, Minus tokens. aziz parents: 24 diff changeset	602 c = *++p;
9c866208b3f6 - Added code for parsing PlusAssign, PlusPlus, Plus, MinusAssign, MinusMinus, Minus tokens. aziz parents: 24 diff changeset	603 if (c == '=')
9c866208b3f6 - Added code for parsing PlusAssign, PlusPlus, Plus, MinusAssign, MinusMinus, Minus tokens. aziz parents: 24 diff changeset	604 t.type = TOK.MinusAssign;
9c866208b3f6 - Added code for parsing PlusAssign, PlusPlus, Plus, MinusAssign, MinusMinus, Minus tokens. aziz parents: 24 diff changeset	605 else if (c == '-')
9c866208b3f6 - Added code for parsing PlusAssign, PlusPlus, Plus, MinusAssign, MinusMinus, Minus tokens. aziz parents: 24 diff changeset	606 t.type = TOK.MinusMinus;
26 c3d7373db241 - Added code for parsing Assign and Equal tokens. aziz parents: 25 diff changeset	607 else {
c3d7373db241 - Added code for parsing Assign and Equal tokens. aziz parents: 25 diff changeset	608 t.type = TOK.Minus;
c3d7373db241 - Added code for parsing Assign and Equal tokens. aziz parents: 25 diff changeset	609 goto Lcommon2;
c3d7373db241 - Added code for parsing Assign and Equal tokens. aziz parents: 25 diff changeset	610 }
c3d7373db241 - Added code for parsing Assign and Equal tokens. aziz parents: 25 diff changeset	611 goto Lcommon;
36 3c7210a722f7 - Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens. aziz parents: 35 diff changeset	612 case '=': /* = == */
26 c3d7373db241 - Added code for parsing Assign and Equal tokens. aziz parents: 25 diff changeset	613 if (p[1] == '=') {
c3d7373db241 - Added code for parsing Assign and Equal tokens. aziz parents: 25 diff changeset	614 ++p;
c3d7373db241 - Added code for parsing Assign and Equal tokens. aziz parents: 25 diff changeset	615 t.type = TOK.Equal;
c3d7373db241 - Added code for parsing Assign and Equal tokens. aziz parents: 25 diff changeset	616 }
25 9c866208b3f6 - Added code for parsing PlusAssign, PlusPlus, Plus, MinusAssign, MinusMinus, Minus tokens. aziz parents: 24 diff changeset	617 else
26 c3d7373db241 - Added code for parsing Assign and Equal tokens. aziz parents: 25 diff changeset	618 t.type = TOK.Assign;
25 9c866208b3f6 - Added code for parsing PlusAssign, PlusPlus, Plus, MinusAssign, MinusMinus, Minus tokens. aziz parents: 24 diff changeset	619 goto Lcommon;
36 3c7210a722f7 - Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens. aziz parents: 35 diff changeset	620 case '~': /* ~ ~= */
27 43b6bf56f0e9 - Added code for parsing CatAssign and Tilde tokens. aziz parents: 26 diff changeset	621 if (p[1] == '=') {
43b6bf56f0e9 - Added code for parsing CatAssign and Tilde tokens. aziz parents: 26 diff changeset	622 ++p;
43b6bf56f0e9 - Added code for parsing CatAssign and Tilde tokens. aziz parents: 26 diff changeset	623 t.type = TOK.CatAssign;
43b6bf56f0e9 - Added code for parsing CatAssign and Tilde tokens. aziz parents: 26 diff changeset	624 }
43b6bf56f0e9 - Added code for parsing CatAssign and Tilde tokens. aziz parents: 26 diff changeset	625 else
43b6bf56f0e9 - Added code for parsing CatAssign and Tilde tokens. aziz parents: 26 diff changeset	626 t.type = TOK.Tilde;
43b6bf56f0e9 - Added code for parsing CatAssign and Tilde tokens. aziz parents: 26 diff changeset	627 goto Lcommon;
36 3c7210a722f7 - Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens. aziz parents: 35 diff changeset	628 case '': / * = /
29 ef83eea26bbd - Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens. aziz parents: 28 diff changeset	629 if (p[1] == '=') {
ef83eea26bbd - Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens. aziz parents: 28 diff changeset	630 ++p;
ef83eea26bbd - Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens. aziz parents: 28 diff changeset	631 t.type = TOK.MulAssign;
ef83eea26bbd - Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens. aziz parents: 28 diff changeset	632 }
ef83eea26bbd - Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens. aziz parents: 28 diff changeset	633 else
ef83eea26bbd - Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens. aziz parents: 28 diff changeset	634 t.type = TOK.Mul;
ef83eea26bbd - Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens. aziz parents: 28 diff changeset	635 goto Lcommon;
36 3c7210a722f7 - Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens. aziz parents: 35 diff changeset	636 case '^': /* ^ ^= */
29 ef83eea26bbd - Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens. aziz parents: 28 diff changeset	637 if (p[1] == '=') {
ef83eea26bbd - Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens. aziz parents: 28 diff changeset	638 ++p;
ef83eea26bbd - Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens. aziz parents: 28 diff changeset	639 t.type = TOK.XorAssign;
ef83eea26bbd - Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens. aziz parents: 28 diff changeset	640 }
ef83eea26bbd - Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens. aziz parents: 28 diff changeset	641 else
ef83eea26bbd - Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens. aziz parents: 28 diff changeset	642 t.type = TOK.Xor;
ef83eea26bbd - Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens. aziz parents: 28 diff changeset	643 goto Lcommon;
36 3c7210a722f7 - Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens. aziz parents: 35 diff changeset	644 case '%': /* % %= */
29 ef83eea26bbd - Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens. aziz parents: 28 diff changeset	645 if (p[1] == '=') {
ef83eea26bbd - Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens. aziz parents: 28 diff changeset	646 ++p;
ef83eea26bbd - Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens. aziz parents: 28 diff changeset	647 t.type = TOK.ModAssign;
ef83eea26bbd - Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens. aziz parents: 28 diff changeset	648 }
ef83eea26bbd - Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens. aziz parents: 28 diff changeset	649 else
ef83eea26bbd - Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens. aziz parents: 28 diff changeset	650 t.type = TOK.Mod;
ef83eea26bbd - Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens. aziz parents: 28 diff changeset	651 goto Lcommon;
ef83eea26bbd - Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens. aziz parents: 28 diff changeset	652 // Single character tokens:
20 d6adfbd7c513 - Added code for parsing braces. aziz parents: 19 diff changeset	653 case '(':
d6adfbd7c513 - Added code for parsing braces. aziz parents: 19 diff changeset	654 t.type = TOK.LParen;
d6adfbd7c513 - Added code for parsing braces. aziz parents: 19 diff changeset	655 goto Lcommon;
d6adfbd7c513 - Added code for parsing braces. aziz parents: 19 diff changeset	656 case ')':
d6adfbd7c513 - Added code for parsing braces. aziz parents: 19 diff changeset	657 t.type = TOK.RParen;
d6adfbd7c513 - Added code for parsing braces. aziz parents: 19 diff changeset	658 goto Lcommon;
d6adfbd7c513 - Added code for parsing braces. aziz parents: 19 diff changeset	659 case '[':
d6adfbd7c513 - Added code for parsing braces. aziz parents: 19 diff changeset	660 t.type = TOK.LBracket;
d6adfbd7c513 - Added code for parsing braces. aziz parents: 19 diff changeset	661 goto Lcommon;
d6adfbd7c513 - Added code for parsing braces. aziz parents: 19 diff changeset	662 case ']':
d6adfbd7c513 - Added code for parsing braces. aziz parents: 19 diff changeset	663 t.type = TOK.RBracket;
d6adfbd7c513 - Added code for parsing braces. aziz parents: 19 diff changeset	664 goto Lcommon;
d6adfbd7c513 - Added code for parsing braces. aziz parents: 19 diff changeset	665 case '{':
d6adfbd7c513 - Added code for parsing braces. aziz parents: 19 diff changeset	666 t.type = TOK.LBrace;
d6adfbd7c513 - Added code for parsing braces. aziz parents: 19 diff changeset	667 goto Lcommon;
d6adfbd7c513 - Added code for parsing braces. aziz parents: 19 diff changeset	668 case '}':
d6adfbd7c513 - Added code for parsing braces. aziz parents: 19 diff changeset	669 t.type = TOK.RBrace;
21 c785c122e4e6 - Added code for parsing Colon, Semicolon, Question, Comma and Dollar. aziz parents: 20 diff changeset	670 goto Lcommon;
c785c122e4e6 - Added code for parsing Colon, Semicolon, Question, Comma and Dollar. aziz parents: 20 diff changeset	671 case ':':
c785c122e4e6 - Added code for parsing Colon, Semicolon, Question, Comma and Dollar. aziz parents: 20 diff changeset	672 t.type = TOK.Colon;
c785c122e4e6 - Added code for parsing Colon, Semicolon, Question, Comma and Dollar. aziz parents: 20 diff changeset	673 goto Lcommon;
c785c122e4e6 - Added code for parsing Colon, Semicolon, Question, Comma and Dollar. aziz parents: 20 diff changeset	674 case ';':
c785c122e4e6 - Added code for parsing Colon, Semicolon, Question, Comma and Dollar. aziz parents: 20 diff changeset	675 t.type = TOK.Semicolon;
c785c122e4e6 - Added code for parsing Colon, Semicolon, Question, Comma and Dollar. aziz parents: 20 diff changeset	676 goto Lcommon;
c785c122e4e6 - Added code for parsing Colon, Semicolon, Question, Comma and Dollar. aziz parents: 20 diff changeset	677 case '?':
c785c122e4e6 - Added code for parsing Colon, Semicolon, Question, Comma and Dollar. aziz parents: 20 diff changeset	678 t.type = TOK.Question;
c785c122e4e6 - Added code for parsing Colon, Semicolon, Question, Comma and Dollar. aziz parents: 20 diff changeset	679 goto Lcommon;
c785c122e4e6 - Added code for parsing Colon, Semicolon, Question, Comma and Dollar. aziz parents: 20 diff changeset	680 case ',':
c785c122e4e6 - Added code for parsing Colon, Semicolon, Question, Comma and Dollar. aziz parents: 20 diff changeset	681 t.type = TOK.Comma;
c785c122e4e6 - Added code for parsing Colon, Semicolon, Question, Comma and Dollar. aziz parents: 20 diff changeset	682 goto Lcommon;
c785c122e4e6 - Added code for parsing Colon, Semicolon, Question, Comma and Dollar. aziz parents: 20 diff changeset	683 case '$':
c785c122e4e6 - Added code for parsing Colon, Semicolon, Question, Comma and Dollar. aziz parents: 20 diff changeset	684 t.type = TOK.Dollar;
20 d6adfbd7c513 - Added code for parsing braces. aziz parents: 19 diff changeset	685 Lcommon:
d6adfbd7c513 - Added code for parsing braces. aziz parents: 19 diff changeset	686 ++p;
26 c3d7373db241 - Added code for parsing Assign and Equal tokens. aziz parents: 25 diff changeset	687 Lcommon2:
20 d6adfbd7c513 - Added code for parsing braces. aziz parents: 19 diff changeset	688 t.end = p;
d6adfbd7c513 - Added code for parsing braces. aziz parents: 19 diff changeset	689 return;
30 426767b94635 - Added code for parsing the '#line' special token. aziz parents: 29 diff changeset	690 case '#':
426767b94635 - Added code for parsing the '#line' special token. aziz parents: 29 diff changeset	691 scanSpecialToken();
51 cadd2bfe686c - Displaying error messages in XML. aziz parents: 50 diff changeset	692 c = *p;
cadd2bfe686c - Displaying error messages in XML. aziz parents: 50 diff changeset	693 continue;
20 d6adfbd7c513 - Added code for parsing braces. aziz parents: 19 diff changeset	694 default:
d6adfbd7c513 - Added code for parsing braces. aziz parents: 19 diff changeset	695 }
d6adfbd7c513 - Added code for parsing braces. aziz parents: 19 diff changeset	696
49 7f0fa15dcffc - Renamed function. aziz parents: 48 diff changeset	697 if (c & 128 && isUniAlpha(decodeUTF8()))
12 0989206cf73c - Added code to decode Unicode characters in identifiers. aziz parents: 11 diff changeset	698 goto Lidentifier;
5 79b4e8848794 - Started writing XML generator. aziz parents: 4 diff changeset	699 c = *++p;
4 92df59b1ec4a - Started implementation of scan(). aziz parents: 3 diff changeset	700 }
3 4bbce78bfb1e - Added TOK enum. aziz parents: 2 diff changeset	701 }
4 92df59b1ec4a - Started implementation of scan(). aziz parents: 3 diff changeset	702
30 426767b94635 - Added code for parsing the '#line' special token. aziz parents: 29 diff changeset	703 void peek(ref Token t)
426767b94635 - Added code for parsing the '#line' special token. aziz parents: 29 diff changeset	704 {
426767b94635 - Added code for parsing the '#line' special token. aziz parents: 29 diff changeset	705 char* tmp = p;
51 cadd2bfe686c - Displaying error messages in XML. aziz parents: 50 diff changeset	706 uint len = errors.length;
30 426767b94635 - Added code for parsing the '#line' special token. aziz parents: 29 diff changeset	707 scan(t);
426767b94635 - Added code for parsing the '#line' special token. aziz parents: 29 diff changeset	708 p = tmp;
51 cadd2bfe686c - Displaying error messages in XML. aziz parents: 50 diff changeset	709 if (errors.length != len)
cadd2bfe686c - Displaying error messages in XML. aziz parents: 50 diff changeset	710 errors = errors[0..len];
30 426767b94635 - Added code for parsing the '#line' special token. aziz parents: 29 diff changeset	711 }
426767b94635 - Added code for parsing the '#line' special token. aziz parents: 29 diff changeset	712
47 8aa37a78937b - Properly implemented scanner for normal string literals. aziz parents: 46 diff changeset	713 void scanNormalStringLiteral(ref Token t)
8aa37a78937b - Properly implemented scanner for normal string literals. aziz parents: 46 diff changeset	714 {
8aa37a78937b - Properly implemented scanner for normal string literals. aziz parents: 46 diff changeset	715 assert(*p == '"');
8aa37a78937b - Properly implemented scanner for normal string literals. aziz parents: 46 diff changeset	716 ++p;
8aa37a78937b - Properly implemented scanner for normal string literals. aziz parents: 46 diff changeset	717 char[] buffer;
8aa37a78937b - Properly implemented scanner for normal string literals. aziz parents: 46 diff changeset	718 t.type = TOK.String;
8aa37a78937b - Properly implemented scanner for normal string literals. aziz parents: 46 diff changeset	719 while (1)
8aa37a78937b - Properly implemented scanner for normal string literals. aziz parents: 46 diff changeset	720 {
8aa37a78937b - Properly implemented scanner for normal string literals. aziz parents: 46 diff changeset	721 switch (*p)
8aa37a78937b - Properly implemented scanner for normal string literals. aziz parents: 46 diff changeset	722 {
8aa37a78937b - Properly implemented scanner for normal string literals. aziz parents: 46 diff changeset	723 case '"':
8aa37a78937b - Properly implemented scanner for normal string literals. aziz parents: 46 diff changeset	724 ++p;
8aa37a78937b - Properly implemented scanner for normal string literals. aziz parents: 46 diff changeset	725 Lreturn:
8aa37a78937b - Properly implemented scanner for normal string literals. aziz parents: 46 diff changeset	726 buffer ~= 0;
49 7f0fa15dcffc - Renamed function. aziz parents: 48 diff changeset	727 t.str = buffer;
47 8aa37a78937b - Properly implemented scanner for normal string literals. aziz parents: 46 diff changeset	728 t.pf = scanPostfix();
8aa37a78937b - Properly implemented scanner for normal string literals. aziz parents: 46 diff changeset	729 t.end = p;
8aa37a78937b - Properly implemented scanner for normal string literals. aziz parents: 46 diff changeset	730 return;
8aa37a78937b - Properly implemented scanner for normal string literals. aziz parents: 46 diff changeset	731 case '\\':
8aa37a78937b - Properly implemented scanner for normal string literals. aziz parents: 46 diff changeset	732 ++p;
49 7f0fa15dcffc - Renamed function. aziz parents: 48 diff changeset	733 dchar d = scanEscapeSequence();
50 4a27b7840ea9 - Return error code 0xFFFF from scanEscapeSequence(). aziz parents: 49 diff changeset	734 if (d == 0xFFFF)
4a27b7840ea9 - Return error code 0xFFFF from scanEscapeSequence(). aziz parents: 49 diff changeset	735 continue;
49 7f0fa15dcffc - Renamed function. aziz parents: 48 diff changeset	736 if (d < 128)
7f0fa15dcffc - Renamed function. aziz parents: 48 diff changeset	737 buffer ~= d;
47 8aa37a78937b - Properly implemented scanner for normal string literals. aziz parents: 46 diff changeset	738 else
49 7f0fa15dcffc - Renamed function. aziz parents: 48 diff changeset	739 encodeUTF8(buffer, d);
47 8aa37a78937b - Properly implemented scanner for normal string literals. aziz parents: 46 diff changeset	740 continue;
8aa37a78937b - Properly implemented scanner for normal string literals. aziz parents: 46 diff changeset	741 case '\r':
8aa37a78937b - Properly implemented scanner for normal string literals. aziz parents: 46 diff changeset	742 if (p[1] == '\n')
8aa37a78937b - Properly implemented scanner for normal string literals. aziz parents: 46 diff changeset	743 ++p;
8aa37a78937b - Properly implemented scanner for normal string literals. aziz parents: 46 diff changeset	744 case '\n':
8aa37a78937b - Properly implemented scanner for normal string literals. aziz parents: 46 diff changeset	745 ++p;
8aa37a78937b - Properly implemented scanner for normal string literals. aziz parents: 46 diff changeset	746 ++loc;
8aa37a78937b - Properly implemented scanner for normal string literals. aziz parents: 46 diff changeset	747 buffer ~= '\n';
8aa37a78937b - Properly implemented scanner for normal string literals. aziz parents: 46 diff changeset	748 continue;
8aa37a78937b - Properly implemented scanner for normal string literals. aziz parents: 46 diff changeset	749 case 0, _Z_:
8aa37a78937b - Properly implemented scanner for normal string literals. aziz parents: 46 diff changeset	750 error(MID.UnterminatedString);
8aa37a78937b - Properly implemented scanner for normal string literals. aziz parents: 46 diff changeset	751 goto Lreturn;
8aa37a78937b - Properly implemented scanner for normal string literals. aziz parents: 46 diff changeset	752 default:
8aa37a78937b - Properly implemented scanner for normal string literals. aziz parents: 46 diff changeset	753 if (*p & 128)
8aa37a78937b - Properly implemented scanner for normal string literals. aziz parents: 46 diff changeset	754 {
49 7f0fa15dcffc - Renamed function. aziz parents: 48 diff changeset	755 char* begin = p;
7f0fa15dcffc - Renamed function. aziz parents: 48 diff changeset	756 dchar d = decodeUTF8();
7f0fa15dcffc - Renamed function. aziz parents: 48 diff changeset	757 if (d == LSd \|\| d == PSd)
47 8aa37a78937b - Properly implemented scanner for normal string literals. aziz parents: 46 diff changeset	758 goto case '\n';
49 7f0fa15dcffc - Renamed function. aziz parents: 48 diff changeset	759
7f0fa15dcffc - Renamed function. aziz parents: 48 diff changeset	760 if (d != 0xFFFF)
7f0fa15dcffc - Renamed function. aziz parents: 48 diff changeset	761 {
7f0fa15dcffc - Renamed function. aziz parents: 48 diff changeset	762 ++p;
7f0fa15dcffc - Renamed function. aziz parents: 48 diff changeset	763 buffer ~= begin[0 .. p - begin];
47 8aa37a78937b - Properly implemented scanner for normal string literals. aziz parents: 46 diff changeset	764 }
8aa37a78937b - Properly implemented scanner for normal string literals. aziz parents: 46 diff changeset	765 continue;
8aa37a78937b - Properly implemented scanner for normal string literals. aziz parents: 46 diff changeset	766 }
8aa37a78937b - Properly implemented scanner for normal string literals. aziz parents: 46 diff changeset	767 buffer ~= *p++;
8aa37a78937b - Properly implemented scanner for normal string literals. aziz parents: 46 diff changeset	768 }
8aa37a78937b - Properly implemented scanner for normal string literals. aziz parents: 46 diff changeset	769 }
8aa37a78937b - Properly implemented scanner for normal string literals. aziz parents: 46 diff changeset	770 }
8aa37a78937b - Properly implemented scanner for normal string literals. aziz parents: 46 diff changeset	771
18 c48d2125f1e2 - Moved code for scanning character literals to separate function. aziz parents: 17 diff changeset	772 void scanCharacterLiteral(ref Token t)
c48d2125f1e2 - Moved code for scanning character literals to separate function. aziz parents: 17 diff changeset	773 {
c48d2125f1e2 - Moved code for scanning character literals to separate function. aziz parents: 17 diff changeset	774 assert(*p == '\'');
c48d2125f1e2 - Moved code for scanning character literals to separate function. aziz parents: 17 diff changeset	775 MID id = MID.UnterminatedCharacterLiteral;
46 e0dbd4722f74 - Scanning character literals correctly now. aziz parents: 45 diff changeset	776 ++p;
e0dbd4722f74 - Scanning character literals correctly now. aziz parents: 45 diff changeset	777 switch (*p)
18 c48d2125f1e2 - Moved code for scanning character literals to separate function. aziz parents: 17 diff changeset	778 {
c48d2125f1e2 - Moved code for scanning character literals to separate function. aziz parents: 17 diff changeset	779 case '\\':
c48d2125f1e2 - Moved code for scanning character literals to separate function. aziz parents: 17 diff changeset	780 ++p;
46 e0dbd4722f74 - Scanning character literals correctly now. aziz parents: 45 diff changeset	781 t.dchar_ = scanEscapeSequence();
18 c48d2125f1e2 - Moved code for scanning character literals to separate function. aziz parents: 17 diff changeset	782 break;
c48d2125f1e2 - Moved code for scanning character literals to separate function. aziz parents: 17 diff changeset	783 case '\'':
46 e0dbd4722f74 - Scanning character literals correctly now. aziz parents: 45 diff changeset	784 ++p;
18 c48d2125f1e2 - Moved code for scanning character literals to separate function. aziz parents: 17 diff changeset	785 id = MID.EmptyCharacterLiteral;
46 e0dbd4722f74 - Scanning character literals correctly now. aziz parents: 45 diff changeset	786 case '\n', '\r', 0, _Z_:
18 c48d2125f1e2 - Moved code for scanning character literals to separate function. aziz parents: 17 diff changeset	787 goto Lerr;
c48d2125f1e2 - Moved code for scanning character literals to separate function. aziz parents: 17 diff changeset	788 default:
46 e0dbd4722f74 - Scanning character literals correctly now. aziz parents: 45 diff changeset	789 uint c = *p;
18 c48d2125f1e2 - Moved code for scanning character literals to separate function. aziz parents: 17 diff changeset	790 if (c & 128)
c48d2125f1e2 - Moved code for scanning character literals to separate function. aziz parents: 17 diff changeset	791 {
49 7f0fa15dcffc - Renamed function. aziz parents: 48 diff changeset	792 c = decodeUTF8();
18 c48d2125f1e2 - Moved code for scanning character literals to separate function. aziz parents: 17 diff changeset	793 if (c == LSd \|\| c == PSd)
c48d2125f1e2 - Moved code for scanning character literals to separate function. aziz parents: 17 diff changeset	794 goto Lerr;
c48d2125f1e2 - Moved code for scanning character literals to separate function. aziz parents: 17 diff changeset	795 }
46 e0dbd4722f74 - Scanning character literals correctly now. aziz parents: 45 diff changeset	796 t.dchar_ = c;
e0dbd4722f74 - Scanning character literals correctly now. aziz parents: 45 diff changeset	797 ++p;
18 c48d2125f1e2 - Moved code for scanning character literals to separate function. aziz parents: 17 diff changeset	798 }
c48d2125f1e2 - Moved code for scanning character literals to separate function. aziz parents: 17 diff changeset	799
46 e0dbd4722f74 - Scanning character literals correctly now. aziz parents: 45 diff changeset	800 if (*p == '\'')
e0dbd4722f74 - Scanning character literals correctly now. aziz parents: 45 diff changeset	801 ++p;
e0dbd4722f74 - Scanning character literals correctly now. aziz parents: 45 diff changeset	802 else
e0dbd4722f74 - Scanning character literals correctly now. aziz parents: 45 diff changeset	803 Lerr:
18 c48d2125f1e2 - Moved code for scanning character literals to separate function. aziz parents: 17 diff changeset	804 error(id);
c48d2125f1e2 - Moved code for scanning character literals to separate function. aziz parents: 17 diff changeset	805 t.type = TOK.Character;
c48d2125f1e2 - Moved code for scanning character literals to separate function. aziz parents: 17 diff changeset	806 t.end = p;
c48d2125f1e2 - Moved code for scanning character literals to separate function. aziz parents: 17 diff changeset	807 }
c48d2125f1e2 - Moved code for scanning character literals to separate function. aziz parents: 17 diff changeset	808
33 cf3047cf3cd2 - Added code for parsing back quote and raw strings. aziz parents: 32 diff changeset	809 char scanPostfix()
31 94f09f4e988e - Added struct for strings to Token with 'pf' = postfix. aziz parents: 30 diff changeset	810 {
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix. aziz parents: 30 diff changeset	811 switch (*p)
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix. aziz parents: 30 diff changeset	812 {
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix. aziz parents: 30 diff changeset	813 case 'c':
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix. aziz parents: 30 diff changeset	814 case 'w':
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix. aziz parents: 30 diff changeset	815 case 'd':
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix. aziz parents: 30 diff changeset	816 return *p++;
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix. aziz parents: 30 diff changeset	817 default:
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix. aziz parents: 30 diff changeset	818 return 0;
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix. aziz parents: 30 diff changeset	819 }
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix. aziz parents: 30 diff changeset	820 }
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix. aziz parents: 30 diff changeset	821
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix. aziz parents: 30 diff changeset	822 void scanRawStringLiteral(ref Token t)
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix. aziz parents: 30 diff changeset	823 {
33 cf3047cf3cd2 - Added code for parsing back quote and raw strings. aziz parents: 32 diff changeset	824 uint delim = *p;
cf3047cf3cd2 - Added code for parsing back quote and raw strings. aziz parents: 32 diff changeset	825 assert(delim == '`' \|\| delim == '"' && p[-1] == 'r');
cf3047cf3cd2 - Added code for parsing back quote and raw strings. aziz parents: 32 diff changeset	826 t.type = TOK.String;
cf3047cf3cd2 - Added code for parsing back quote and raw strings. aziz parents: 32 diff changeset	827 char[] buffer;
cf3047cf3cd2 - Added code for parsing back quote and raw strings. aziz parents: 32 diff changeset	828 uint c;
cf3047cf3cd2 - Added code for parsing back quote and raw strings. aziz parents: 32 diff changeset	829 while (1)
cf3047cf3cd2 - Added code for parsing back quote and raw strings. aziz parents: 32 diff changeset	830 {
cf3047cf3cd2 - Added code for parsing back quote and raw strings. aziz parents: 32 diff changeset	831 c = *++p;
cf3047cf3cd2 - Added code for parsing back quote and raw strings. aziz parents: 32 diff changeset	832 switch (c)
cf3047cf3cd2 - Added code for parsing back quote and raw strings. aziz parents: 32 diff changeset	833 {
cf3047cf3cd2 - Added code for parsing back quote and raw strings. aziz parents: 32 diff changeset	834 case '\r':
cf3047cf3cd2 - Added code for parsing back quote and raw strings. aziz parents: 32 diff changeset	835 if (p[1] == '\n')
cf3047cf3cd2 - Added code for parsing back quote and raw strings. aziz parents: 32 diff changeset	836 ++p;
cf3047cf3cd2 - Added code for parsing back quote and raw strings. aziz parents: 32 diff changeset	837 c = '\n'; // Convert '\r' and '\r\n' to '\n'
cf3047cf3cd2 - Added code for parsing back quote and raw strings. aziz parents: 32 diff changeset	838 case '\n':
cf3047cf3cd2 - Added code for parsing back quote and raw strings. aziz parents: 32 diff changeset	839 ++loc;
52 f65a83c27638 - Fixed the raw string literal scanner. Newlines weren't copied to the buffer. Converting LS and PS to '\n' as well. aziz parents: 51 diff changeset	840 break;
33 cf3047cf3cd2 - Added code for parsing back quote and raw strings. aziz parents: 32 diff changeset	841 case '`':
cf3047cf3cd2 - Added code for parsing back quote and raw strings. aziz parents: 32 diff changeset	842 case '"':
cf3047cf3cd2 - Added code for parsing back quote and raw strings. aziz parents: 32 diff changeset	843 if (c == delim)
cf3047cf3cd2 - Added code for parsing back quote and raw strings. aziz parents: 32 diff changeset	844 {
cf3047cf3cd2 - Added code for parsing back quote and raw strings. aziz parents: 32 diff changeset	845 ++p;
cf3047cf3cd2 - Added code for parsing back quote and raw strings. aziz parents: 32 diff changeset	846 t.pf = scanPostfix();
cf3047cf3cd2 - Added code for parsing back quote and raw strings. aziz parents: 32 diff changeset	847 Lreturn:
cf3047cf3cd2 - Added code for parsing back quote and raw strings. aziz parents: 32 diff changeset	848 t.str = buffer ~ '\0';
cf3047cf3cd2 - Added code for parsing back quote and raw strings. aziz parents: 32 diff changeset	849 t.end = p;
cf3047cf3cd2 - Added code for parsing back quote and raw strings. aziz parents: 32 diff changeset	850 return;
cf3047cf3cd2 - Added code for parsing back quote and raw strings. aziz parents: 32 diff changeset	851 }
cf3047cf3cd2 - Added code for parsing back quote and raw strings. aziz parents: 32 diff changeset	852 break;
cf3047cf3cd2 - Added code for parsing back quote and raw strings. aziz parents: 32 diff changeset	853 case LS[0]:
cf3047cf3cd2 - Added code for parsing back quote and raw strings. aziz parents: 32 diff changeset	854 if (p[1] == LS[1] && (p[2] == LS[2] \|\| p[2] == PS[2]))
cf3047cf3cd2 - Added code for parsing back quote and raw strings. aziz parents: 32 diff changeset	855 {
52 f65a83c27638 - Fixed the raw string literal scanner. Newlines weren't copied to the buffer. Converting LS and PS to '\n' as well. aziz parents: 51 diff changeset	856 c = '\n';
f65a83c27638 - Fixed the raw string literal scanner. Newlines weren't copied to the buffer. Converting LS and PS to '\n' as well. aziz parents: 51 diff changeset	857 ++p; ++p;
33 cf3047cf3cd2 - Added code for parsing back quote and raw strings. aziz parents: 32 diff changeset	858 ++loc;
cf3047cf3cd2 - Added code for parsing back quote and raw strings. aziz parents: 32 diff changeset	859 }
cf3047cf3cd2 - Added code for parsing back quote and raw strings. aziz parents: 32 diff changeset	860 break;
cf3047cf3cd2 - Added code for parsing back quote and raw strings. aziz parents: 32 diff changeset	861 case 0, _Z_:
cf3047cf3cd2 - Added code for parsing back quote and raw strings. aziz parents: 32 diff changeset	862 if (delim == 'r')
cf3047cf3cd2 - Added code for parsing back quote and raw strings. aziz parents: 32 diff changeset	863 error(MID.UnterminatedRawString);
cf3047cf3cd2 - Added code for parsing back quote and raw strings. aziz parents: 32 diff changeset	864 else
cf3047cf3cd2 - Added code for parsing back quote and raw strings. aziz parents: 32 diff changeset	865 error(MID.UnterminatedBackQuoteString);
cf3047cf3cd2 - Added code for parsing back quote and raw strings. aziz parents: 32 diff changeset	866 goto Lreturn;
cf3047cf3cd2 - Added code for parsing back quote and raw strings. aziz parents: 32 diff changeset	867 default:
cf3047cf3cd2 - Added code for parsing back quote and raw strings. aziz parents: 32 diff changeset	868 }
cf3047cf3cd2 - Added code for parsing back quote and raw strings. aziz parents: 32 diff changeset	869 buffer ~= c; // copy character to buffer
cf3047cf3cd2 - Added code for parsing back quote and raw strings. aziz parents: 32 diff changeset	870 }
cf3047cf3cd2 - Added code for parsing back quote and raw strings. aziz parents: 32 diff changeset	871 assert(0);
31 94f09f4e988e - Added struct for strings to Token with 'pf' = postfix. aziz parents: 30 diff changeset	872 }
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix. aziz parents: 30 diff changeset	873
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix. aziz parents: 30 diff changeset	874 void scanHexStringLiteral(ref Token t)
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix. aziz parents: 30 diff changeset	875 {
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix. aziz parents: 30 diff changeset	876 assert(p[0] == 'x' && p[1] == '"');
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix. aziz parents: 30 diff changeset	877 t.type = TOK.String;
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix. aziz parents: 30 diff changeset	878
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix. aziz parents: 30 diff changeset	879 uint c;
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix. aziz parents: 30 diff changeset	880 ubyte[] buffer;
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix. aziz parents: 30 diff changeset	881 ubyte h; // hex number
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix. aziz parents: 30 diff changeset	882 uint n; // number of hex digits
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix. aziz parents: 30 diff changeset	883
53 1786c2825491 - Fixed scanner for hex string literals. Terminating string with 0. Relocated some code to the bottom. aziz parents: 52 diff changeset	884 ++p;
31 94f09f4e988e - Added struct for strings to Token with 'pf' = postfix. aziz parents: 30 diff changeset	885 while (1)
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix. aziz parents: 30 diff changeset	886 {
53 1786c2825491 - Fixed scanner for hex string literals. Terminating string with 0. Relocated some code to the bottom. aziz parents: 52 diff changeset	887 c = *++p;
31 94f09f4e988e - Added struct for strings to Token with 'pf' = postfix. aziz parents: 30 diff changeset	888 switch (c)
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix. aziz parents: 30 diff changeset	889 {
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix. aziz parents: 30 diff changeset	890 case '"':
53 1786c2825491 - Fixed scanner for hex string literals. Terminating string with 0. Relocated some code to the bottom. aziz parents: 52 diff changeset	891 ++p;
31 94f09f4e988e - Added struct for strings to Token with 'pf' = postfix. aziz parents: 30 diff changeset	892 if (n & 1)
53 1786c2825491 - Fixed scanner for hex string literals. Terminating string with 0. Relocated some code to the bottom. aziz parents: 52 diff changeset	893 error(MID.OddNumberOfDigitsInHexString);
1786c2825491 - Fixed scanner for hex string literals. Terminating string with 0. Relocated some code to the bottom. aziz parents: 52 diff changeset	894 t.pf = scanPostfix();
1786c2825491 - Fixed scanner for hex string literals. Terminating string with 0. Relocated some code to the bottom. aziz parents: 52 diff changeset	895 Lreturn:
1786c2825491 - Fixed scanner for hex string literals. Terminating string with 0. Relocated some code to the bottom. aziz parents: 52 diff changeset	896 buffer ~= 0;
31 94f09f4e988e - Added struct for strings to Token with 'pf' = postfix. aziz parents: 30 diff changeset	897 t.str = cast(string) buffer;
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix. aziz parents: 30 diff changeset	898 t.end = p;
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix. aziz parents: 30 diff changeset	899 return;
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix. aziz parents: 30 diff changeset	900 case '\r':
53 1786c2825491 - Fixed scanner for hex string literals. Terminating string with 0. Relocated some code to the bottom. aziz parents: 52 diff changeset	901 if (p[1] == '\n')
31 94f09f4e988e - Added struct for strings to Token with 'pf' = postfix. aziz parents: 30 diff changeset	902 ++p;
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix. aziz parents: 30 diff changeset	903 case '\n':
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix. aziz parents: 30 diff changeset	904 ++loc;
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix. aziz parents: 30 diff changeset	905 continue;
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix. aziz parents: 30 diff changeset	906 default:
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix. aziz parents: 30 diff changeset	907 if (ishexad(c))
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix. aziz parents: 30 diff changeset	908 {
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix. aziz parents: 30 diff changeset	909 if (c <= '9')
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix. aziz parents: 30 diff changeset	910 c -= '0';
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix. aziz parents: 30 diff changeset	911 else if (c <= 'F')
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix. aziz parents: 30 diff changeset	912 c -= 'A' - 10;
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix. aziz parents: 30 diff changeset	913 else
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix. aziz parents: 30 diff changeset	914 c -= 'a' - 10;
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix. aziz parents: 30 diff changeset	915
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix. aziz parents: 30 diff changeset	916 if (n & 1)
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix. aziz parents: 30 diff changeset	917 {
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix. aziz parents: 30 diff changeset	918 h <<= 4;
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix. aziz parents: 30 diff changeset	919 h \|= c;
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix. aziz parents: 30 diff changeset	920 buffer ~= h;
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix. aziz parents: 30 diff changeset	921 }
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix. aziz parents: 30 diff changeset	922 else
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix. aziz parents: 30 diff changeset	923 h = c;
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix. aziz parents: 30 diff changeset	924 ++n;
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix. aziz parents: 30 diff changeset	925 continue;
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix. aziz parents: 30 diff changeset	926 }
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix. aziz parents: 30 diff changeset	927 else if (isspace(c))
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix. aziz parents: 30 diff changeset	928 continue;
53 1786c2825491 - Fixed scanner for hex string literals. Terminating string with 0. Relocated some code to the bottom. aziz parents: 52 diff changeset	929 else if (c == LS[0] && p[1] == LS[1] && (p[2] == LS[2] \|\| p[2] == PS[2]))
1786c2825491 - Fixed scanner for hex string literals. Terminating string with 0. Relocated some code to the bottom. aziz parents: 52 diff changeset	930 {
1786c2825491 - Fixed scanner for hex string literals. Terminating string with 0. Relocated some code to the bottom. aziz parents: 52 diff changeset	931 ++p; ++p;
1786c2825491 - Fixed scanner for hex string literals. Terminating string with 0. Relocated some code to the bottom. aziz parents: 52 diff changeset	932 ++loc;
1786c2825491 - Fixed scanner for hex string literals. Terminating string with 0. Relocated some code to the bottom. aziz parents: 52 diff changeset	933 continue;
1786c2825491 - Fixed scanner for hex string literals. Terminating string with 0. Relocated some code to the bottom. aziz parents: 52 diff changeset	934 }
1786c2825491 - Fixed scanner for hex string literals. Terminating string with 0. Relocated some code to the bottom. aziz parents: 52 diff changeset	935 else if (c == 0 \|\| c == _Z_)
1786c2825491 - Fixed scanner for hex string literals. Terminating string with 0. Relocated some code to the bottom. aziz parents: 52 diff changeset	936 {
1786c2825491 - Fixed scanner for hex string literals. Terminating string with 0. Relocated some code to the bottom. aziz parents: 52 diff changeset	937 error(MID.UnterminatedHexString);
1786c2825491 - Fixed scanner for hex string literals. Terminating string with 0. Relocated some code to the bottom. aziz parents: 52 diff changeset	938 t.pf = 0;
1786c2825491 - Fixed scanner for hex string literals. Terminating string with 0. Relocated some code to the bottom. aziz parents: 52 diff changeset	939 goto Lreturn;
1786c2825491 - Fixed scanner for hex string literals. Terminating string with 0. Relocated some code to the bottom. aziz parents: 52 diff changeset	940 }
1786c2825491 - Fixed scanner for hex string literals. Terminating string with 0. Relocated some code to the bottom. aziz parents: 52 diff changeset	941 error(MID.NonHexCharInHexString);
31 94f09f4e988e - Added struct for strings to Token with 'pf' = postfix. aziz parents: 30 diff changeset	942 }
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix. aziz parents: 30 diff changeset	943 }
53 1786c2825491 - Fixed scanner for hex string literals. Terminating string with 0. Relocated some code to the bottom. aziz parents: 52 diff changeset	944 assert(0);
31 94f09f4e988e - Added struct for strings to Token with 'pf' = postfix. aziz parents: 30 diff changeset	945 }
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix. aziz parents: 30 diff changeset	946
45 cc84b9ca9d0a - Implemented escape sequence scanner. aziz parents: 44 diff changeset	947 dchar scanEscapeSequence()
cc84b9ca9d0a - Implemented escape sequence scanner. aziz parents: 44 diff changeset	948 {
cc84b9ca9d0a - Implemented escape sequence scanner. aziz parents: 44 diff changeset	949 uint c = char2ev(*p);
46 e0dbd4722f74 - Scanning character literals correctly now. aziz parents: 45 diff changeset	950 if (c) {
e0dbd4722f74 - Scanning character literals correctly now. aziz parents: 45 diff changeset	951 ++p;
45 cc84b9ca9d0a - Implemented escape sequence scanner. aziz parents: 44 diff changeset	952 return c;
46 e0dbd4722f74 - Scanning character literals correctly now. aziz parents: 45 diff changeset	953 }
50 4a27b7840ea9 - Return error code 0xFFFF from scanEscapeSequence(). aziz parents: 49 diff changeset	954 c = 0xFFFF;
45 cc84b9ca9d0a - Implemented escape sequence scanner. aziz parents: 44 diff changeset	955 uint digits = 2;
cc84b9ca9d0a - Implemented escape sequence scanner. aziz parents: 44 diff changeset	956
cc84b9ca9d0a - Implemented escape sequence scanner. aziz parents: 44 diff changeset	957 switch (*p)
cc84b9ca9d0a - Implemented escape sequence scanner. aziz parents: 44 diff changeset	958 {
cc84b9ca9d0a - Implemented escape sequence scanner. aziz parents: 44 diff changeset	959 case 'x':
cc84b9ca9d0a - Implemented escape sequence scanner. aziz parents: 44 diff changeset	960 c = 0;
cc84b9ca9d0a - Implemented escape sequence scanner. aziz parents: 44 diff changeset	961 while (1)
cc84b9ca9d0a - Implemented escape sequence scanner. aziz parents: 44 diff changeset	962 {
cc84b9ca9d0a - Implemented escape sequence scanner. aziz parents: 44 diff changeset	963 ++p;
cc84b9ca9d0a - Implemented escape sequence scanner. aziz parents: 44 diff changeset	964 if (ishexad(*p))
cc84b9ca9d0a - Implemented escape sequence scanner. aziz parents: 44 diff changeset	965 {
cc84b9ca9d0a - Implemented escape sequence scanner. aziz parents: 44 diff changeset	966 c *= 16;
cc84b9ca9d0a - Implemented escape sequence scanner. aziz parents: 44 diff changeset	967 if (*p <= '9')
48 c2e0e0269c28 - Added code for scanning escape string literals. aziz parents: 47 diff changeset	968 c += *p - '0';
45 cc84b9ca9d0a - Implemented escape sequence scanner. aziz parents: 44 diff changeset	969 else if (*p <= 'F')
48 c2e0e0269c28 - Added code for scanning escape string literals. aziz parents: 47 diff changeset	970 c += *p - 'A' + 10;
45 cc84b9ca9d0a - Implemented escape sequence scanner. aziz parents: 44 diff changeset	971 else
48 c2e0e0269c28 - Added code for scanning escape string literals. aziz parents: 47 diff changeset	972 c += *p - 'a' + 10;
c2e0e0269c28 - Added code for scanning escape string literals. aziz parents: 47 diff changeset	973 if (!--digits) {
c2e0e0269c28 - Added code for scanning escape string literals. aziz parents: 47 diff changeset	974 ++p;
45 cc84b9ca9d0a - Implemented escape sequence scanner. aziz parents: 44 diff changeset	975 break;
48 c2e0e0269c28 - Added code for scanning escape string literals. aziz parents: 47 diff changeset	976 }
45 cc84b9ca9d0a - Implemented escape sequence scanner. aziz parents: 44 diff changeset	977 }
cc84b9ca9d0a - Implemented escape sequence scanner. aziz parents: 44 diff changeset	978 else
cc84b9ca9d0a - Implemented escape sequence scanner. aziz parents: 44 diff changeset	979 {
cc84b9ca9d0a - Implemented escape sequence scanner. aziz parents: 44 diff changeset	980 error(MID.InsufficientHexDigits);
50 4a27b7840ea9 - Return error code 0xFFFF from scanEscapeSequence(). aziz parents: 49 diff changeset	981 c = 0xFFFF;
45 cc84b9ca9d0a - Implemented escape sequence scanner. aziz parents: 44 diff changeset	982 break;
cc84b9ca9d0a - Implemented escape sequence scanner. aziz parents: 44 diff changeset	983 }
cc84b9ca9d0a - Implemented escape sequence scanner. aziz parents: 44 diff changeset	984 }
cc84b9ca9d0a - Implemented escape sequence scanner. aziz parents: 44 diff changeset	985 break;
cc84b9ca9d0a - Implemented escape sequence scanner. aziz parents: 44 diff changeset	986 case 'u':
cc84b9ca9d0a - Implemented escape sequence scanner. aziz parents: 44 diff changeset	987 digits = 4;
cc84b9ca9d0a - Implemented escape sequence scanner. aziz parents: 44 diff changeset	988 goto case 'x';
cc84b9ca9d0a - Implemented escape sequence scanner. aziz parents: 44 diff changeset	989 case 'U':
cc84b9ca9d0a - Implemented escape sequence scanner. aziz parents: 44 diff changeset	990 digits = 8;
cc84b9ca9d0a - Implemented escape sequence scanner. aziz parents: 44 diff changeset	991 goto case 'x';
cc84b9ca9d0a - Implemented escape sequence scanner. aziz parents: 44 diff changeset	992 default:
48 c2e0e0269c28 - Added code for scanning escape string literals. aziz parents: 47 diff changeset	993 if (isoctal(*p))
45 cc84b9ca9d0a - Implemented escape sequence scanner. aziz parents: 44 diff changeset	994 {
48 c2e0e0269c28 - Added code for scanning escape string literals. aziz parents: 47 diff changeset	995 c = 0;
c2e0e0269c28 - Added code for scanning escape string literals. aziz parents: 47 diff changeset	996 c += *p - '0';
c2e0e0269c28 - Added code for scanning escape string literals. aziz parents: 47 diff changeset	997 ++p;
c2e0e0269c28 - Added code for scanning escape string literals. aziz parents: 47 diff changeset	998 if (!isoctal(*p))
c2e0e0269c28 - Added code for scanning escape string literals. aziz parents: 47 diff changeset	999 return c;
c2e0e0269c28 - Added code for scanning escape string literals. aziz parents: 47 diff changeset	1000 c *= 8;
c2e0e0269c28 - Added code for scanning escape string literals. aziz parents: 47 diff changeset	1001 c += *p - '0';
c2e0e0269c28 - Added code for scanning escape string literals. aziz parents: 47 diff changeset	1002 ++p;
c2e0e0269c28 - Added code for scanning escape string literals. aziz parents: 47 diff changeset	1003 if (!isoctal(*p))
c2e0e0269c28 - Added code for scanning escape string literals. aziz parents: 47 diff changeset	1004 return c;
c2e0e0269c28 - Added code for scanning escape string literals. aziz parents: 47 diff changeset	1005 c *= 8;
c2e0e0269c28 - Added code for scanning escape string literals. aziz parents: 47 diff changeset	1006 c += *p - '0';
c2e0e0269c28 - Added code for scanning escape string literals. aziz parents: 47 diff changeset	1007 ++p;
c2e0e0269c28 - Added code for scanning escape string literals. aziz parents: 47 diff changeset	1008 }
c2e0e0269c28 - Added code for scanning escape string literals. aziz parents: 47 diff changeset	1009 else if(*p == '&')
c2e0e0269c28 - Added code for scanning escape string literals. aziz parents: 47 diff changeset	1010 {
c2e0e0269c28 - Added code for scanning escape string literals. aziz parents: 47 diff changeset	1011 if (isalpha(*++p))
45 cc84b9ca9d0a - Implemented escape sequence scanner. aziz parents: 44 diff changeset	1012 {
48 c2e0e0269c28 - Added code for scanning escape string literals. aziz parents: 47 diff changeset	1013 while (1)
c2e0e0269c28 - Added code for scanning escape string literals. aziz parents: 47 diff changeset	1014 {
c2e0e0269c28 - Added code for scanning escape string literals. aziz parents: 47 diff changeset	1015 if (isalnum(*++p))
c2e0e0269c28 - Added code for scanning escape string literals. aziz parents: 47 diff changeset	1016 continue;
c2e0e0269c28 - Added code for scanning escape string literals. aziz parents: 47 diff changeset	1017 if (*p == ';') {
c2e0e0269c28 - Added code for scanning escape string literals. aziz parents: 47 diff changeset	1018 // TODO: convert entity to unicode codepoint.
c2e0e0269c28 - Added code for scanning escape string literals. aziz parents: 47 diff changeset	1019 ++p;
c2e0e0269c28 - Added code for scanning escape string literals. aziz parents: 47 diff changeset	1020 break;
c2e0e0269c28 - Added code for scanning escape string literals. aziz parents: 47 diff changeset	1021 }
c2e0e0269c28 - Added code for scanning escape string literals. aziz parents: 47 diff changeset	1022 else {
c2e0e0269c28 - Added code for scanning escape string literals. aziz parents: 47 diff changeset	1023 error(MID.UnterminatedHTMLEntity);
c2e0e0269c28 - Added code for scanning escape string literals. aziz parents: 47 diff changeset	1024 break;
c2e0e0269c28 - Added code for scanning escape string literals. aziz parents: 47 diff changeset	1025 }
45 cc84b9ca9d0a - Implemented escape sequence scanner. aziz parents: 44 diff changeset	1026 }
cc84b9ca9d0a - Implemented escape sequence scanner. aziz parents: 44 diff changeset	1027 }
48 c2e0e0269c28 - Added code for scanning escape string literals. aziz parents: 47 diff changeset	1028 else
c2e0e0269c28 - Added code for scanning escape string literals. aziz parents: 47 diff changeset	1029 error(MID.InvalidBeginHTMLEntity);
45 cc84b9ca9d0a - Implemented escape sequence scanner. aziz parents: 44 diff changeset	1030 }
cc84b9ca9d0a - Implemented escape sequence scanner. aziz parents: 44 diff changeset	1031 else
48 c2e0e0269c28 - Added code for scanning escape string literals. aziz parents: 47 diff changeset	1032 error(MID.UndefinedEscapeSequence);
45 cc84b9ca9d0a - Implemented escape sequence scanner. aziz parents: 44 diff changeset	1033 }
cc84b9ca9d0a - Implemented escape sequence scanner. aziz parents: 44 diff changeset	1034
cc84b9ca9d0a - Implemented escape sequence scanner. aziz parents: 44 diff changeset	1035 return c;
cc84b9ca9d0a - Implemented escape sequence scanner. aziz parents: 44 diff changeset	1036 }
cc84b9ca9d0a - Implemented escape sequence scanner. aziz parents: 44 diff changeset	1037
15 c70c028e47dd - Started implementation of lexing numbers. aziz parents: 14 diff changeset	1038 void scanNumber(ref Token t)
c70c028e47dd - Started implementation of lexing numbers. aziz parents: 14 diff changeset	1039 {
c70c028e47dd - Started implementation of lexing numbers. aziz parents: 14 diff changeset	1040 while (isdigit(*++p)) {}
c70c028e47dd - Started implementation of lexing numbers. aziz parents: 14 diff changeset	1041 t.type = TOK.Number;
c70c028e47dd - Started implementation of lexing numbers. aziz parents: 14 diff changeset	1042 t.end = p;
30 426767b94635 - Added code for parsing the '#line' special token. aziz parents: 29 diff changeset	1043 t._uint = toInt(t.span);
426767b94635 - Added code for parsing the '#line' special token. aziz parents: 29 diff changeset	1044 }
426767b94635 - Added code for parsing the '#line' special token. aziz parents: 29 diff changeset	1045
426767b94635 - Added code for parsing the '#line' special token. aziz parents: 29 diff changeset	1046 /// Scan special token: #line Integer [Filespec] EndOfLine
51 cadd2bfe686c - Displaying error messages in XML. aziz parents: 50 diff changeset	1047 // TODO: Handle case like: #line 0 #line 2
30 426767b94635 - Added code for parsing the '#line' special token. aziz parents: 29 diff changeset	1048 void scanSpecialToken()
426767b94635 - Added code for parsing the '#line' special token. aziz parents: 29 diff changeset	1049 {
47 8aa37a78937b - Properly implemented scanner for normal string literals. aziz parents: 46 diff changeset	1050 assert(*p == '#');
51 cadd2bfe686c - Displaying error messages in XML. aziz parents: 50 diff changeset	1051
47 8aa37a78937b - Properly implemented scanner for normal string literals. aziz parents: 46 diff changeset	1052 ++p;
30 426767b94635 - Added code for parsing the '#line' special token. aziz parents: 29 diff changeset	1053 MID mid;
426767b94635 - Added code for parsing the '#line' special token. aziz parents: 29 diff changeset	1054 Token t;
51 cadd2bfe686c - Displaying error messages in XML. aziz parents: 50 diff changeset	1055 uint oldloc = this.loc, newloc;
30 426767b94635 - Added code for parsing the '#line' special token. aziz parents: 29 diff changeset	1056
51 cadd2bfe686c - Displaying error messages in XML. aziz parents: 50 diff changeset	1057 peek(t);
cadd2bfe686c - Displaying error messages in XML. aziz parents: 50 diff changeset	1058 if (!(this.loc == oldloc && p == t.start && t.type == TOK.Identifier && t.span == "line"))
cadd2bfe686c - Displaying error messages in XML. aziz parents: 50 diff changeset	1059 {
cadd2bfe686c - Displaying error messages in XML. aziz parents: 50 diff changeset	1060 this.loc = oldloc; // reset this.loc because we took a peek at the next token
cadd2bfe686c - Displaying error messages in XML. aziz parents: 50 diff changeset	1061 mid = MID.ExpectedIdentifierSTLine;
cadd2bfe686c - Displaying error messages in XML. aziz parents: 50 diff changeset	1062 goto Lerr;
cadd2bfe686c - Displaying error messages in XML. aziz parents: 50 diff changeset	1063 }
cadd2bfe686c - Displaying error messages in XML. aziz parents: 50 diff changeset	1064 p = t.end; // consume token
cadd2bfe686c - Displaying error messages in XML. aziz parents: 50 diff changeset	1065
cadd2bfe686c - Displaying error messages in XML. aziz parents: 50 diff changeset	1066 peek(t);
cadd2bfe686c - Displaying error messages in XML. aziz parents: 50 diff changeset	1067 if (this.loc == oldloc && t.type == TOK.Number)
cadd2bfe686c - Displaying error messages in XML. aziz parents: 50 diff changeset	1068 {
cadd2bfe686c - Displaying error messages in XML. aziz parents: 50 diff changeset	1069 newloc = t._uint - 1;
cadd2bfe686c - Displaying error messages in XML. aziz parents: 50 diff changeset	1070 p = t.end;
cadd2bfe686c - Displaying error messages in XML. aziz parents: 50 diff changeset	1071 }
cadd2bfe686c - Displaying error messages in XML. aziz parents: 50 diff changeset	1072 else
cadd2bfe686c - Displaying error messages in XML. aziz parents: 50 diff changeset	1073 {
cadd2bfe686c - Displaying error messages in XML. aziz parents: 50 diff changeset	1074 this.loc = oldloc;
cadd2bfe686c - Displaying error messages in XML. aziz parents: 50 diff changeset	1075 mid = MID.ExpectedNumberAfterSTLine;
30 426767b94635 - Added code for parsing the '#line' special token. aziz parents: 29 diff changeset	1076 goto Lerr;
426767b94635 - Added code for parsing the '#line' special token. aziz parents: 29 diff changeset	1077 }
426767b94635 - Added code for parsing the '#line' special token. aziz parents: 29 diff changeset	1078
51 cadd2bfe686c - Displaying error messages in XML. aziz parents: 50 diff changeset	1079 peek(t);
cadd2bfe686c - Displaying error messages in XML. aziz parents: 50 diff changeset	1080 if (this.loc != oldloc)
30 426767b94635 - Added code for parsing the '#line' special token. aziz parents: 29 diff changeset	1081 {
51 cadd2bfe686c - Displaying error messages in XML. aziz parents: 50 diff changeset	1082 this.loc = oldloc;
cadd2bfe686c - Displaying error messages in XML. aziz parents: 50 diff changeset	1083 mid = MID.NewlineInSpecialToken;
cadd2bfe686c - Displaying error messages in XML. aziz parents: 50 diff changeset	1084 goto Lerr;
30 426767b94635 - Added code for parsing the '#line' special token. aziz parents: 29 diff changeset	1085 }
426767b94635 - Added code for parsing the '#line' special token. aziz parents: 29 diff changeset	1086 if (t.type == TOK.String)
426767b94635 - Added code for parsing the '#line' special token. aziz parents: 29 diff changeset	1087 {
51 cadd2bfe686c - Displaying error messages in XML. aziz parents: 50 diff changeset	1088 if (*t.start != '"')
cadd2bfe686c - Displaying error messages in XML. aziz parents: 50 diff changeset	1089 {
cadd2bfe686c - Displaying error messages in XML. aziz parents: 50 diff changeset	1090 mid = MID.ExpectedNormalStringLiteral;
30 426767b94635 - Added code for parsing the '#line' special token. aziz parents: 29 diff changeset	1091 goto Lerr;
51 cadd2bfe686c - Displaying error messages in XML. aziz parents: 50 diff changeset	1092 }
30 426767b94635 - Added code for parsing the '#line' special token. aziz parents: 29 diff changeset	1093 fileName = t.span[1..$-1]; // contents of "..."
426767b94635 - Added code for parsing the '#line' special token. aziz parents: 29 diff changeset	1094 p = t.end;
426767b94635 - Added code for parsing the '#line' special token. aziz parents: 29 diff changeset	1095 }
426767b94635 - Added code for parsing the '#line' special token. aziz parents: 29 diff changeset	1096 else if (t.type == TOK.Identifier && t.span == "__FILE__")
426767b94635 - Added code for parsing the '#line' special token. aziz parents: 29 diff changeset	1097 {
426767b94635 - Added code for parsing the '#line' special token. aziz parents: 29 diff changeset	1098 p = t.end;
426767b94635 - Added code for parsing the '#line' special token. aziz parents: 29 diff changeset	1099 }
51 cadd2bfe686c - Displaying error messages in XML. aziz parents: 50 diff changeset	1100 /+
cadd2bfe686c - Displaying error messages in XML. aziz parents: 50 diff changeset	1101 peek(t);
cadd2bfe686c - Displaying error messages in XML. aziz parents: 50 diff changeset	1102 if (this.loc == oldloc && t.type != TOK.EOF)
cadd2bfe686c - Displaying error messages in XML. aziz parents: 50 diff changeset	1103 {
cadd2bfe686c - Displaying error messages in XML. aziz parents: 50 diff changeset	1104 mid = MID.UnterminatedSpecialToken;
cadd2bfe686c - Displaying error messages in XML. aziz parents: 50 diff changeset	1105 goto Lerr;
cadd2bfe686c - Displaying error messages in XML. aziz parents: 50 diff changeset	1106 }
cadd2bfe686c - Displaying error messages in XML. aziz parents: 50 diff changeset	1107 +/
30 426767b94635 - Added code for parsing the '#line' special token. aziz parents: 29 diff changeset	1108 while (1)
426767b94635 - Added code for parsing the '#line' special token. aziz parents: 29 diff changeset	1109 {
51 cadd2bfe686c - Displaying error messages in XML. aziz parents: 50 diff changeset	1110 switch (*p)
cadd2bfe686c - Displaying error messages in XML. aziz parents: 50 diff changeset	1111 {
cadd2bfe686c - Displaying error messages in XML. aziz parents: 50 diff changeset	1112 case '\r':
cadd2bfe686c - Displaying error messages in XML. aziz parents: 50 diff changeset	1113 if (p[1] == '\n')
cadd2bfe686c - Displaying error messages in XML. aziz parents: 50 diff changeset	1114 ++p;
cadd2bfe686c - Displaying error messages in XML. aziz parents: 50 diff changeset	1115 case '\n':
cadd2bfe686c - Displaying error messages in XML. aziz parents: 50 diff changeset	1116 ++p;
30 426767b94635 - Added code for parsing the '#line' special token. aziz parents: 29 diff changeset	1117 break;
51 cadd2bfe686c - Displaying error messages in XML. aziz parents: 50 diff changeset	1118 case LS[0]:
cadd2bfe686c - Displaying error messages in XML. aziz parents: 50 diff changeset	1119 if (p[1] == LS[1] && (p[2] == LS[2] \|\| p[2] == PS[2]))
cadd2bfe686c - Displaying error messages in XML. aziz parents: 50 diff changeset	1120 {
cadd2bfe686c - Displaying error messages in XML. aziz parents: 50 diff changeset	1121 p += 2;
cadd2bfe686c - Displaying error messages in XML. aziz parents: 50 diff changeset	1122 break;
cadd2bfe686c - Displaying error messages in XML. aziz parents: 50 diff changeset	1123 }
cadd2bfe686c - Displaying error messages in XML. aziz parents: 50 diff changeset	1124 case 0, _Z_:
cadd2bfe686c - Displaying error messages in XML. aziz parents: 50 diff changeset	1125 break;
cadd2bfe686c - Displaying error messages in XML. aziz parents: 50 diff changeset	1126 default:
cadd2bfe686c - Displaying error messages in XML. aziz parents: 50 diff changeset	1127 if (isspace(*p)) {
cadd2bfe686c - Displaying error messages in XML. aziz parents: 50 diff changeset	1128 ++p;
cadd2bfe686c - Displaying error messages in XML. aziz parents: 50 diff changeset	1129 continue;
cadd2bfe686c - Displaying error messages in XML. aziz parents: 50 diff changeset	1130 }
30 426767b94635 - Added code for parsing the '#line' special token. aziz parents: 29 diff changeset	1131 mid = MID.UnterminatedSpecialToken;
426767b94635 - Added code for parsing the '#line' special token. aziz parents: 29 diff changeset	1132 goto Lerr;
426767b94635 - Added code for parsing the '#line' special token. aziz parents: 29 diff changeset	1133 }
51 cadd2bfe686c - Displaying error messages in XML. aziz parents: 50 diff changeset	1134 break;
30 426767b94635 - Added code for parsing the '#line' special token. aziz parents: 29 diff changeset	1135 }
426767b94635 - Added code for parsing the '#line' special token. aziz parents: 29 diff changeset	1136
51 cadd2bfe686c - Displaying error messages in XML. aziz parents: 50 diff changeset	1137 this.loc = newloc;
30 426767b94635 - Added code for parsing the '#line' special token. aziz parents: 29 diff changeset	1138 return;
426767b94635 - Added code for parsing the '#line' special token. aziz parents: 29 diff changeset	1139 Lerr:
426767b94635 - Added code for parsing the '#line' special token. aziz parents: 29 diff changeset	1140 error(mid);
15 c70c028e47dd - Started implementation of lexing numbers. aziz parents: 14 diff changeset	1141 }
c70c028e47dd - Started implementation of lexing numbers. aziz parents: 14 diff changeset	1142
49 7f0fa15dcffc - Renamed function. aziz parents: 48 diff changeset	1143 uint decodeUTF8()
12 0989206cf73c - Added code to decode Unicode characters in identifiers. aziz parents: 11 diff changeset	1144 {
49 7f0fa15dcffc - Renamed function. aziz parents: 48 diff changeset	1145 assert(*p & 128, "check for ASCII char before calling decodeUTF8().");
12 0989206cf73c - Added code to decode Unicode characters in identifiers. aziz parents: 11 diff changeset	1146 size_t idx;
49 7f0fa15dcffc - Renamed function. aziz parents: 48 diff changeset	1147 uint d = 0xFFFF;
7f0fa15dcffc - Renamed function. aziz parents: 48 diff changeset	1148 try
7f0fa15dcffc - Renamed function. aziz parents: 48 diff changeset	1149 {
7f0fa15dcffc - Renamed function. aziz parents: 48 diff changeset	1150 d = std.utf.decode(p[0 .. end-p], idx);
7f0fa15dcffc - Renamed function. aziz parents: 48 diff changeset	1151 p += idx -1;
7f0fa15dcffc - Renamed function. aziz parents: 48 diff changeset	1152 }
7f0fa15dcffc - Renamed function. aziz parents: 48 diff changeset	1153 catch (UtfException e)
7f0fa15dcffc - Renamed function. aziz parents: 48 diff changeset	1154 {
7f0fa15dcffc - Renamed function. aziz parents: 48 diff changeset	1155 error(MID.InvalidUTF8Sequence);
7f0fa15dcffc - Renamed function. aziz parents: 48 diff changeset	1156 // Skip to next valid utf-8 sequence
7f0fa15dcffc - Renamed function. aziz parents: 48 diff changeset	1157 while (UTF8stride[*++p] != 0xFF) {}
7f0fa15dcffc - Renamed function. aziz parents: 48 diff changeset	1158 }
12 0989206cf73c - Added code to decode Unicode characters in identifiers. aziz parents: 11 diff changeset	1159 return d;
0989206cf73c - Added code to decode Unicode characters in identifiers. aziz parents: 11 diff changeset	1160 }
0989206cf73c - Added code to decode Unicode characters in identifiers. aziz parents: 11 diff changeset	1161
28 3a9daccf7d96 - Added table for identifiers to Lexer. aziz parents: 27 diff changeset	1162 void loadKeywords()
3a9daccf7d96 - Added table for identifiers to Lexer. aziz parents: 27 diff changeset	1163 {
3a9daccf7d96 - Added table for identifiers to Lexer. aziz parents: 27 diff changeset	1164 foreach(k; keywords)
3a9daccf7d96 - Added table for identifiers to Lexer. aziz parents: 27 diff changeset	1165 idtable[k.str] = k;
3a9daccf7d96 - Added table for identifiers to Lexer. aziz parents: 27 diff changeset	1166 }
3a9daccf7d96 - Added table for identifiers to Lexer. aziz parents: 27 diff changeset	1167
18 c48d2125f1e2 - Moved code for scanning character literals to separate function. aziz parents: 17 diff changeset	1168 void error(MID id)
c48d2125f1e2 - Moved code for scanning character literals to separate function. aziz parents: 17 diff changeset	1169 {
c48d2125f1e2 - Moved code for scanning character literals to separate function. aziz parents: 17 diff changeset	1170 errors ~= new Problem(Problem.Type.Lexer, id, loc);
c48d2125f1e2 - Moved code for scanning character literals to separate function. aziz parents: 17 diff changeset	1171 }
c48d2125f1e2 - Moved code for scanning character literals to separate function. aziz parents: 17 diff changeset	1172
3 4bbce78bfb1e - Added TOK enum. aziz parents: 2 diff changeset	1173 public TOK nextToken()
4bbce78bfb1e - Added TOK enum. aziz parents: 2 diff changeset	1174 {
4 92df59b1ec4a - Started implementation of scan(). aziz parents: 3 diff changeset	1175 scan(this.token);
92df59b1ec4a - Started implementation of scan(). aziz parents: 3 diff changeset	1176 return this.token.type;
92df59b1ec4a - Started implementation of scan(). aziz parents: 3 diff changeset	1177 }
92df59b1ec4a - Started implementation of scan(). aziz parents: 3 diff changeset	1178
92df59b1ec4a - Started implementation of scan(). aziz parents: 3 diff changeset	1179 Token[] getTokens()
92df59b1ec4a - Started implementation of scan(). aziz parents: 3 diff changeset	1180 {
92df59b1ec4a - Started implementation of scan(). aziz parents: 3 diff changeset	1181 Token[] tokens;
92df59b1ec4a - Started implementation of scan(). aziz parents: 3 diff changeset	1182 while (nextToken() != TOK.EOF)
92df59b1ec4a - Started implementation of scan(). aziz parents: 3 diff changeset	1183 tokens ~= this.token;
92df59b1ec4a - Started implementation of scan(). aziz parents: 3 diff changeset	1184 tokens ~= this.token;
92df59b1ec4a - Started implementation of scan(). aziz parents: 3 diff changeset	1185 return tokens;
3 4bbce78bfb1e - Added TOK enum. aziz parents: 2 diff changeset	1186 }
48 c2e0e0269c28 - Added code for scanning escape string literals. aziz parents: 47 diff changeset	1187
c2e0e0269c28 - Added code for scanning escape string literals. aziz parents: 47 diff changeset	1188 private void encodeUTF8(inout char[] str, dchar d)
c2e0e0269c28 - Added code for scanning escape string literals. aziz parents: 47 diff changeset	1189 {
c2e0e0269c28 - Added code for scanning escape string literals. aziz parents: 47 diff changeset	1190 char[6] b;
c2e0e0269c28 - Added code for scanning escape string literals. aziz parents: 47 diff changeset	1191 assert(d > 0x7F, "check for ASCII char before calling encodeUTF8().");
c2e0e0269c28 - Added code for scanning escape string literals. aziz parents: 47 diff changeset	1192 if (d < 0x800)
c2e0e0269c28 - Added code for scanning escape string literals. aziz parents: 47 diff changeset	1193 {
c2e0e0269c28 - Added code for scanning escape string literals. aziz parents: 47 diff changeset	1194 b[0] = 0xC0 \| (d >> 6);
c2e0e0269c28 - Added code for scanning escape string literals. aziz parents: 47 diff changeset	1195 b[1] = 0x80 \| (d & 0x3F);
c2e0e0269c28 - Added code for scanning escape string literals. aziz parents: 47 diff changeset	1196 str ~= b[0..2];
c2e0e0269c28 - Added code for scanning escape string literals. aziz parents: 47 diff changeset	1197 }
c2e0e0269c28 - Added code for scanning escape string literals. aziz parents: 47 diff changeset	1198 else if (d < 0x10000)
c2e0e0269c28 - Added code for scanning escape string literals. aziz parents: 47 diff changeset	1199 {
c2e0e0269c28 - Added code for scanning escape string literals. aziz parents: 47 diff changeset	1200 b[0] = 0xE0 \| (d >> 12);
c2e0e0269c28 - Added code for scanning escape string literals. aziz parents: 47 diff changeset	1201 b[1] = 0x80 \| ((d >> 6) & 0x3F);
c2e0e0269c28 - Added code for scanning escape string literals. aziz parents: 47 diff changeset	1202 b[2] = 0x80 \| (d & 0x3F);
c2e0e0269c28 - Added code for scanning escape string literals. aziz parents: 47 diff changeset	1203 str ~= b[0..3];
c2e0e0269c28 - Added code for scanning escape string literals. aziz parents: 47 diff changeset	1204 }
c2e0e0269c28 - Added code for scanning escape string literals. aziz parents: 47 diff changeset	1205 else if (d < 0x200000)
c2e0e0269c28 - Added code for scanning escape string literals. aziz parents: 47 diff changeset	1206 {
c2e0e0269c28 - Added code for scanning escape string literals. aziz parents: 47 diff changeset	1207 b[0] = 0xF0 \| (d >> 18);
c2e0e0269c28 - Added code for scanning escape string literals. aziz parents: 47 diff changeset	1208 b[1] = 0x80 \| ((d >> 12) & 0x3F);
c2e0e0269c28 - Added code for scanning escape string literals. aziz parents: 47 diff changeset	1209 b[2] = 0x80 \| ((d >> 6) & 0x3F);
c2e0e0269c28 - Added code for scanning escape string literals. aziz parents: 47 diff changeset	1210 b[3] = 0x80 \| (d & 0x3F);
c2e0e0269c28 - Added code for scanning escape string literals. aziz parents: 47 diff changeset	1211 str ~= b[0..4];
c2e0e0269c28 - Added code for scanning escape string literals. aziz parents: 47 diff changeset	1212 }
c2e0e0269c28 - Added code for scanning escape string literals. aziz parents: 47 diff changeset	1213 else if (d < 0x4000000)
c2e0e0269c28 - Added code for scanning escape string literals. aziz parents: 47 diff changeset	1214 {
c2e0e0269c28 - Added code for scanning escape string literals. aziz parents: 47 diff changeset	1215 b[0] = 0xF8 \| (d >> 24);
c2e0e0269c28 - Added code for scanning escape string literals. aziz parents: 47 diff changeset	1216 b[1] = 0x80 \| ((d >> 18) & 0x3F);
c2e0e0269c28 - Added code for scanning escape string literals. aziz parents: 47 diff changeset	1217 b[2] = 0x80 \| ((d >> 12) & 0x3F);
c2e0e0269c28 - Added code for scanning escape string literals. aziz parents: 47 diff changeset	1218 b[3] = 0x80 \| ((d >> 6) & 0x3F);
c2e0e0269c28 - Added code for scanning escape string literals. aziz parents: 47 diff changeset	1219 b[4] = 0x80 \| (d & 0x3F);
c2e0e0269c28 - Added code for scanning escape string literals. aziz parents: 47 diff changeset	1220 str ~= b[0..5];
c2e0e0269c28 - Added code for scanning escape string literals. aziz parents: 47 diff changeset	1221 }
c2e0e0269c28 - Added code for scanning escape string literals. aziz parents: 47 diff changeset	1222 else if (d < 0x80000000)
c2e0e0269c28 - Added code for scanning escape string literals. aziz parents: 47 diff changeset	1223 {
c2e0e0269c28 - Added code for scanning escape string literals. aziz parents: 47 diff changeset	1224 b[0] = 0xFC \| (d >> 30);
c2e0e0269c28 - Added code for scanning escape string literals. aziz parents: 47 diff changeset	1225 b[1] = 0x80 \| ((d >> 24) & 0x3F);
c2e0e0269c28 - Added code for scanning escape string literals. aziz parents: 47 diff changeset	1226 b[2] = 0x80 \| ((d >> 18) & 0x3F);
c2e0e0269c28 - Added code for scanning escape string literals. aziz parents: 47 diff changeset	1227 b[3] = 0x80 \| ((d >> 12) & 0x3F);
c2e0e0269c28 - Added code for scanning escape string literals. aziz parents: 47 diff changeset	1228 b[4] = 0x80 \| ((d >> 6) & 0x3F);
c2e0e0269c28 - Added code for scanning escape string literals. aziz parents: 47 diff changeset	1229 b[5] = 0x80 \| (d & 0x3F);
c2e0e0269c28 - Added code for scanning escape string literals. aziz parents: 47 diff changeset	1230 str ~= b[0..6];
c2e0e0269c28 - Added code for scanning escape string literals. aziz parents: 47 diff changeset	1231 }
c2e0e0269c28 - Added code for scanning escape string literals. aziz parents: 47 diff changeset	1232 else
c2e0e0269c28 - Added code for scanning escape string literals. aziz parents: 47 diff changeset	1233 error(MID.InvalidUnicodeCharacter);
c2e0e0269c28 - Added code for scanning escape string literals. aziz parents: 47 diff changeset	1234 }
5 79b4e8848794 - Started writing XML generator. aziz parents: 4 diff changeset	1235 }
39 69b940398d7b - Added unittest to test correct parsing of operator tokens. aziz parents: 38 diff changeset	1236
69b940398d7b - Added unittest to test correct parsing of operator tokens. aziz parents: 38 diff changeset	1237 unittest
69b940398d7b - Added unittest to test correct parsing of operator tokens. aziz parents: 38 diff changeset	1238 {
40 9d5ceb0f8be9 - Added more tokens for testing. aziz parents: 39 diff changeset	1239 string[] toks = [
9d5ceb0f8be9 - Added more tokens for testing. aziz parents: 39 diff changeset	1240 ">", ">=", ">>", ">>=", ">>>", ">>>=", "<", "<=", "<>",
9d5ceb0f8be9 - Added more tokens for testing. aziz parents: 39 diff changeset	1241 "<>=", "<<", "<<=", "!", "!<", "!>", "!<=", "!>=", "!<>",
9d5ceb0f8be9 - Added more tokens for testing. aziz parents: 39 diff changeset	1242 "!<>=", ".", "..", "...", "&", "&&", "&=", "+", "++",
9d5ceb0f8be9 - Added more tokens for testing. aziz parents: 39 diff changeset	1243 "+=", "-", "--", "-=", "=", "==", "~", "~=", "*",
9d5ceb0f8be9 - Added more tokens for testing. aziz parents: 39 diff changeset	1244 "*=", "/", "/=", "^", "^=", "%", "%=", "(", ")",
9d5ceb0f8be9 - Added more tokens for testing. aziz parents: 39 diff changeset	1245 "[", "]", "{", "}", ":", ";", "?", ",", "$"
9d5ceb0f8be9 - Added more tokens for testing. aziz parents: 39 diff changeset	1246 ];
39 69b940398d7b - Added unittest to test correct parsing of operator tokens. aziz parents: 38 diff changeset	1247
69b940398d7b - Added unittest to test correct parsing of operator tokens. aziz parents: 38 diff changeset	1248 char[] src;
69b940398d7b - Added unittest to test correct parsing of operator tokens. aziz parents: 38 diff changeset	1249
40 9d5ceb0f8be9 - Added more tokens for testing. aziz parents: 39 diff changeset	1250 foreach (op; toks)
39 69b940398d7b - Added unittest to test correct parsing of operator tokens. aziz parents: 38 diff changeset	1251 src ~= op ~ " ";
69b940398d7b - Added unittest to test correct parsing of operator tokens. aziz parents: 38 diff changeset	1252
69b940398d7b - Added unittest to test correct parsing of operator tokens. aziz parents: 38 diff changeset	1253 auto lx = new Lexer(src, "");
69b940398d7b - Added unittest to test correct parsing of operator tokens. aziz parents: 38 diff changeset	1254 auto tokens = lx.getTokens();
69b940398d7b - Added unittest to test correct parsing of operator tokens. aziz parents: 38 diff changeset	1255
69b940398d7b - Added unittest to test correct parsing of operator tokens. aziz parents: 38 diff changeset	1256 tokens = tokens[0..$-1]; // exclude TOK.EOF
69b940398d7b - Added unittest to test correct parsing of operator tokens. aziz parents: 38 diff changeset	1257
40 9d5ceb0f8be9 - Added more tokens for testing. aziz parents: 39 diff changeset	1258 assert(tokens.length == toks.length );
39 69b940398d7b - Added unittest to test correct parsing of operator tokens. aziz parents: 38 diff changeset	1259
69b940398d7b - Added unittest to test correct parsing of operator tokens. aziz parents: 38 diff changeset	1260 foreach (i, t; tokens)
40 9d5ceb0f8be9 - Added more tokens for testing. aziz parents: 39 diff changeset	1261 assert(t.span == toks[i], std.string.format("Lexed '%s' but expected '%s'", t.span, toks[i]));
41 2b7be1d67d4d - Optimized scanner of block comments. aziz parents: 40 diff changeset	1262 }

Mercurial > projects > dil

annotate trunk/src/Lexer.d @ 53:1786c2825491