projects/dil: trunk/src/Lexer.d annotate

annotate trunk/src/Lexer.d @ 33:cf3047cf3cd2

- Added code for parsing back quote and raw strings. - Made F lower-case in scanPostFix(). - Added more operators to be output in <op> tags.

author	aziz
date	Mon, 25 Jun 2007 19:04:03 +0000
parents	d7011daa4740
children	41dad4aef4b1

rev	line source
0 8ba2570de175 Initial import. aziz parents: diff changeset	1 /++
8ba2570de175 Initial import. aziz parents: diff changeset	2 Author: Aziz Köksal
8ba2570de175 Initial import. aziz parents: diff changeset	3 License: GPL2
8ba2570de175 Initial import. aziz parents: diff changeset	4 +/
8ba2570de175 Initial import. aziz parents: diff changeset	5 module Lexer;
3 4bbce78bfb1e - Added TOK enum. aziz parents: 2 diff changeset	6 import Token;
28 3a9daccf7d96 - Added table for identifiers to Lexer. aziz parents: 27 diff changeset	7 import Keywords;
3a9daccf7d96 - Added table for identifiers to Lexer. aziz parents: 27 diff changeset	8 import Identifier;
2 81c6cc33f5c8 - Initializing ptable with a precomputed array literal. aziz parents: 1 diff changeset	9 import std.stdio;
4 92df59b1ec4a - Started implementation of scan(). aziz parents: 3 diff changeset	10 import std.utf;
92df59b1ec4a - Started implementation of scan(). aziz parents: 3 diff changeset	11 import std.uni;
30 426767b94635 - Added code for parsing the '#line' special token. aziz parents: 29 diff changeset	12 import std.conv;
0 8ba2570de175 Initial import. aziz parents: diff changeset	13
8ba2570de175 Initial import. aziz parents: diff changeset	14 /// ASCII character properties table.
2 81c6cc33f5c8 - Initializing ptable with a precomputed array literal. aziz parents: 1 diff changeset	15 static const int ptable[256] = [
16 476e8e55c1d4 - Added Whitespace to the character properties table. aziz parents: 15 diff changeset	16 0, 0, 0, 0, 0, 0, 0, 0, 0,32, 0,32,32, 0, 0, 0,
13 e5211758b63c - Added isidbeg() function. aziz parents: 12 diff changeset	17 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
16 476e8e55c1d4 - Added Whitespace to the character properties table. aziz parents: 15 diff changeset	18 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
13 e5211758b63c - Added isidbeg() function. aziz parents: 12 diff changeset	19 7, 7, 7, 7, 7, 7, 7, 7, 6, 6, 0, 0, 0, 0, 0, 0,
e5211758b63c - Added isidbeg() function. aziz parents: 12 diff changeset	20 0,12,12,12,12,12,12, 8, 8, 8, 8, 8, 8, 8, 8, 8,
e5211758b63c - Added isidbeg() function. aziz parents: 12 diff changeset	21 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 0, 0, 0, 0,16,
e5211758b63c - Added isidbeg() function. aziz parents: 12 diff changeset	22 0,12,12,12,12,12,12, 8, 8, 8, 8, 8, 8, 8, 8, 8,
e5211758b63c - Added isidbeg() function. aziz parents: 12 diff changeset	23 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 0, 0, 0, 0, 0,
e5211758b63c - Added isidbeg() function. aziz parents: 12 diff changeset	24 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
e5211758b63c - Added isidbeg() function. aziz parents: 12 diff changeset	25 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
e5211758b63c - Added isidbeg() function. aziz parents: 12 diff changeset	26 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
e5211758b63c - Added isidbeg() function. aziz parents: 12 diff changeset	27 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
e5211758b63c - Added isidbeg() function. aziz parents: 12 diff changeset	28 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
e5211758b63c - Added isidbeg() function. aziz parents: 12 diff changeset	29 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
e5211758b63c - Added isidbeg() function. aziz parents: 12 diff changeset	30 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
e5211758b63c - Added isidbeg() function. aziz parents: 12 diff changeset	31 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
2 81c6cc33f5c8 - Initializing ptable with a precomputed array literal. aziz parents: 1 diff changeset	32 ];
0 8ba2570de175 Initial import. aziz parents: diff changeset	33
8ba2570de175 Initial import. aziz parents: diff changeset	34 enum CProperty
8ba2570de175 Initial import. aziz parents: diff changeset	35 {
1 f3cd3bfde4ba - Corrected some errors to make the file compile. aziz parents: 0 diff changeset	36 Octal = 1,
0 8ba2570de175 Initial import. aziz parents: diff changeset	37 Digit = 1<<1,
8ba2570de175 Initial import. aziz parents: diff changeset	38 Hex = 1<<2,
8ba2570de175 Initial import. aziz parents: diff changeset	39 Alpha = 1<<3,
13 e5211758b63c - Added isidbeg() function. aziz parents: 12 diff changeset	40 Underscore = 1<<4,
16 476e8e55c1d4 - Added Whitespace to the character properties table. aziz parents: 15 diff changeset	41 Whitespace = 1<<5
0 8ba2570de175 Initial import. aziz parents: diff changeset	42 }
8ba2570de175 Initial import. aziz parents: diff changeset	43
13 e5211758b63c - Added isidbeg() function. aziz parents: 12 diff changeset	44 private alias CProperty CP;
e5211758b63c - Added isidbeg() function. aziz parents: 12 diff changeset	45 int isoctal(char c) { return ptable[c] & CP.Octal; }
e5211758b63c - Added isidbeg() function. aziz parents: 12 diff changeset	46 int isdigit(char c) { return ptable[c] & CP.Digit; }
e5211758b63c - Added isidbeg() function. aziz parents: 12 diff changeset	47 int ishexad(char c) { return ptable[c] & CP.Hex; }
e5211758b63c - Added isidbeg() function. aziz parents: 12 diff changeset	48 int isalpha(char c) { return ptable[c] & CP.Alpha; }
e5211758b63c - Added isidbeg() function. aziz parents: 12 diff changeset	49 int isalnum(char c) { return ptable[c] & (CP.Alpha \| CP.Digit); }
e5211758b63c - Added isidbeg() function. aziz parents: 12 diff changeset	50 int isidbeg(char c) { return ptable[c] & (CP.Alpha \| CP.Underscore); }
e5211758b63c - Added isidbeg() function. aziz parents: 12 diff changeset	51 int isident(char c) { return ptable[c] & (CP.Alpha \| CP.Underscore \| CP.Digit); }
16 476e8e55c1d4 - Added Whitespace to the character properties table. aziz parents: 15 diff changeset	52 int isspace(char c) { return ptable[c] & CP.Whitespace; }
13 e5211758b63c - Added isidbeg() function. aziz parents: 12 diff changeset	53
e5211758b63c - Added isidbeg() function. aziz parents: 12 diff changeset	54 version(gen_ptable)
0 8ba2570de175 Initial import. aziz parents: diff changeset	55 static this()
8ba2570de175 Initial import. aziz parents: diff changeset	56 {
2 81c6cc33f5c8 - Initializing ptable with a precomputed array literal. aziz parents: 1 diff changeset	57 // Initialize character properties table.
0 8ba2570de175 Initial import. aziz parents: diff changeset	58 for (int i; i < ptable.length; ++i)
8ba2570de175 Initial import. aziz parents: diff changeset	59 {
13 e5211758b63c - Added isidbeg() function. aziz parents: 12 diff changeset	60 ptable[i] = 0;
0 8ba2570de175 Initial import. aziz parents: diff changeset	61 if ('0' <= i && i <= '7')
13 e5211758b63c - Added isidbeg() function. aziz parents: 12 diff changeset	62 ptable[i] \|= CP.Octal;
0 8ba2570de175 Initial import. aziz parents: diff changeset	63 if ('0' <= i && i <= '9')
13 e5211758b63c - Added isidbeg() function. aziz parents: 12 diff changeset	64 ptable[i] \|= CP.Digit;
0 8ba2570de175 Initial import. aziz parents: diff changeset	65 if (isdigit(i) \|\| 'a' <= i && i <= 'f' \|\| 'A' <= i && i <= 'F')
13 e5211758b63c - Added isidbeg() function. aziz parents: 12 diff changeset	66 ptable[i] \|= CP.Hex;
0 8ba2570de175 Initial import. aziz parents: diff changeset	67 if ('a' <= i && i <= 'z' \|\| 'A' <= i && i <= 'Z')
13 e5211758b63c - Added isidbeg() function. aziz parents: 12 diff changeset	68 ptable[i] \|= CP.Alpha;
e5211758b63c - Added isidbeg() function. aziz parents: 12 diff changeset	69 if (i == '_')
e5211758b63c - Added isidbeg() function. aziz parents: 12 diff changeset	70 ptable[i] \|= CP.Underscore;
16 476e8e55c1d4 - Added Whitespace to the character properties table. aziz parents: 15 diff changeset	71 if (i == ' ' \|\| i == '\t' \|\| i == '\v'\|\| i == '\f')
476e8e55c1d4 - Added Whitespace to the character properties table. aziz parents: 15 diff changeset	72 ptable[i] \|= CP.Whitespace;
0 8ba2570de175 Initial import. aziz parents: diff changeset	73 }
2 81c6cc33f5c8 - Initializing ptable with a precomputed array literal. aziz parents: 1 diff changeset	74 // Print a formatted array literal.
81c6cc33f5c8 - Initializing ptable with a precomputed array literal. aziz parents: 1 diff changeset	75 char[] array = "[\n";
81c6cc33f5c8 - Initializing ptable with a precomputed array literal. aziz parents: 1 diff changeset	76 for (int i; i < ptable.length; ++i)
81c6cc33f5c8 - Initializing ptable with a precomputed array literal. aziz parents: 1 diff changeset	77 {
13 e5211758b63c - Added isidbeg() function. aziz parents: 12 diff changeset	78 int c = ptable[i];
e5211758b63c - Added isidbeg() function. aziz parents: 12 diff changeset	79 array ~= std.string.format("%2d,", c, ((i+1) % 16) ? "":"\n");
2 81c6cc33f5c8 - Initializing ptable with a precomputed array literal. aziz parents: 1 diff changeset	80 }
13 e5211758b63c - Added isidbeg() function. aziz parents: 12 diff changeset	81 array[$-2..$] = "\n]";
2 81c6cc33f5c8 - Initializing ptable with a precomputed array literal. aziz parents: 1 diff changeset	82 writefln(array);
0 8ba2570de175 Initial import. aziz parents: diff changeset	83 }
8ba2570de175 Initial import. aziz parents: diff changeset	84
11 dffcdaa7c47a - Added Unicode line and paragraph separators. aziz parents: 10 diff changeset	85 const char[3] LS = \u2028;
dffcdaa7c47a - Added Unicode line and paragraph separators. aziz parents: 10 diff changeset	86 const char[3] PS = \u2029;
dffcdaa7c47a - Added Unicode line and paragraph separators. aziz parents: 10 diff changeset	87
18 c48d2125f1e2 - Moved code for scanning character literals to separate function. aziz parents: 17 diff changeset	88 const dchar LSd = 0x2028;
c48d2125f1e2 - Moved code for scanning character literals to separate function. aziz parents: 17 diff changeset	89 const dchar PSd = 0x2029;
c48d2125f1e2 - Moved code for scanning character literals to separate function. aziz parents: 17 diff changeset	90
32 d7011daa4740 - Added missing commas after the items in the messages table. aziz parents: 31 diff changeset	91 const uint _Z_ = 26; /// Control+Z
d7011daa4740 - Added missing commas after the items in the messages table. aziz parents: 31 diff changeset	92
18 c48d2125f1e2 - Moved code for scanning character literals to separate function. aziz parents: 17 diff changeset	93 /// Index into table of error messages.
c48d2125f1e2 - Moved code for scanning character literals to separate function. aziz parents: 17 diff changeset	94 enum MID
c48d2125f1e2 - Moved code for scanning character literals to separate function. aziz parents: 17 diff changeset	95 {
c48d2125f1e2 - Moved code for scanning character literals to separate function. aziz parents: 17 diff changeset	96 UnterminatedCharacterLiteral,
30 426767b94635 - Added code for parsing the '#line' special token. aziz parents: 29 diff changeset	97 EmptyCharacterLiteral,
31 94f09f4e988e - Added struct for strings to Token with 'pf' = postfix. aziz parents: 30 diff changeset	98 // #line
30 426767b94635 - Added code for parsing the '#line' special token. aziz parents: 29 diff changeset	99 ExpectedIdentifierLine,
426767b94635 - Added code for parsing the '#line' special token. aziz parents: 29 diff changeset	100 NewlineInSpecialToken,
31 94f09f4e988e - Added struct for strings to Token with 'pf' = postfix. aziz parents: 30 diff changeset	101 UnterminatedSpecialToken,
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix. aziz parents: 30 diff changeset	102 // x""
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix. aziz parents: 30 diff changeset	103 NonHexCharInHexString,
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix. aziz parents: 30 diff changeset	104 OddNumberOfDigitsInHexString,
32 d7011daa4740 - Added missing commas after the items in the messages table. aziz parents: 31 diff changeset	105 UnterminatedHexString,
d7011daa4740 - Added missing commas after the items in the messages table. aziz parents: 31 diff changeset	106 // /* */ /+ +/
d7011daa4740 - Added missing commas after the items in the messages table. aziz parents: 31 diff changeset	107 UnterminatedBlockComment,
33 cf3047cf3cd2 - Added code for parsing back quote and raw strings. aziz parents: 32 diff changeset	108 UnterminatedNestedComment,
cf3047cf3cd2 - Added code for parsing back quote and raw strings. aziz parents: 32 diff changeset	109 // `` r""
cf3047cf3cd2 - Added code for parsing back quote and raw strings. aziz parents: 32 diff changeset	110 UnterminatedRawString,
cf3047cf3cd2 - Added code for parsing back quote and raw strings. aziz parents: 32 diff changeset	111 UnterminatedBackQuoteString,
18 c48d2125f1e2 - Moved code for scanning character literals to separate function. aziz parents: 17 diff changeset	112 }
c48d2125f1e2 - Moved code for scanning character literals to separate function. aziz parents: 17 diff changeset	113
31 94f09f4e988e - Added struct for strings to Token with 'pf' = postfix. aziz parents: 30 diff changeset	114 string[] messages = [
32 d7011daa4740 - Added missing commas after the items in the messages table. aziz parents: 31 diff changeset	115 "unterminated character literal.",
30 426767b94635 - Added code for parsing the '#line' special token. aziz parents: 29 diff changeset	116 "empty character literal.",
31 94f09f4e988e - Added struct for strings to Token with 'pf' = postfix. aziz parents: 30 diff changeset	117 // #line
32 d7011daa4740 - Added missing commas after the items in the messages table. aziz parents: 31 diff changeset	118 "expected 'line' after '#'.",
d7011daa4740 - Added missing commas after the items in the messages table. aziz parents: 31 diff changeset	119 "newline not allowed inside special token.",
31 94f09f4e988e - Added struct for strings to Token with 'pf' = postfix. aziz parents: 30 diff changeset	120 "expected newline after special token.",
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix. aziz parents: 30 diff changeset	121 // x""
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix. aziz parents: 30 diff changeset	122 "non-hex character '{1}' found in hex string.",
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix. aziz parents: 30 diff changeset	123 "odd number of hex digits in hex string.",
32 d7011daa4740 - Added missing commas after the items in the messages table. aziz parents: 31 diff changeset	124 "unterminated hex string.",
d7011daa4740 - Added missing commas after the items in the messages table. aziz parents: 31 diff changeset	125 // /* */ /+ +/
d7011daa4740 - Added missing commas after the items in the messages table. aziz parents: 31 diff changeset	126 "unterminated block comment (/* */).",
33 cf3047cf3cd2 - Added code for parsing back quote and raw strings. aziz parents: 32 diff changeset	127 "unterminated nested comment (/+ +/).",
cf3047cf3cd2 - Added code for parsing back quote and raw strings. aziz parents: 32 diff changeset	128 // `` r""
cf3047cf3cd2 - Added code for parsing back quote and raw strings. aziz parents: 32 diff changeset	129 "unterminated raw string.",
cf3047cf3cd2 - Added code for parsing back quote and raw strings. aziz parents: 32 diff changeset	130 "unterminated back quote string.",
18 c48d2125f1e2 - Moved code for scanning character literals to separate function. aziz parents: 17 diff changeset	131 ];
c48d2125f1e2 - Moved code for scanning character literals to separate function. aziz parents: 17 diff changeset	132
c48d2125f1e2 - Moved code for scanning character literals to separate function. aziz parents: 17 diff changeset	133 class Problem
c48d2125f1e2 - Moved code for scanning character literals to separate function. aziz parents: 17 diff changeset	134 {
c48d2125f1e2 - Moved code for scanning character literals to separate function. aziz parents: 17 diff changeset	135 enum Type
c48d2125f1e2 - Moved code for scanning character literals to separate function. aziz parents: 17 diff changeset	136 {
c48d2125f1e2 - Moved code for scanning character literals to separate function. aziz parents: 17 diff changeset	137 Lexer,
c48d2125f1e2 - Moved code for scanning character literals to separate function. aziz parents: 17 diff changeset	138 Parser,
c48d2125f1e2 - Moved code for scanning character literals to separate function. aziz parents: 17 diff changeset	139 Semantic
c48d2125f1e2 - Moved code for scanning character literals to separate function. aziz parents: 17 diff changeset	140 }
c48d2125f1e2 - Moved code for scanning character literals to separate function. aziz parents: 17 diff changeset	141
c48d2125f1e2 - Moved code for scanning character literals to separate function. aziz parents: 17 diff changeset	142 MID id;
c48d2125f1e2 - Moved code for scanning character literals to separate function. aziz parents: 17 diff changeset	143 Type type;
c48d2125f1e2 - Moved code for scanning character literals to separate function. aziz parents: 17 diff changeset	144 uint loc;
c48d2125f1e2 - Moved code for scanning character literals to separate function. aziz parents: 17 diff changeset	145 this(Type type, MID id, uint loc)
c48d2125f1e2 - Moved code for scanning character literals to separate function. aziz parents: 17 diff changeset	146 {
c48d2125f1e2 - Moved code for scanning character literals to separate function. aziz parents: 17 diff changeset	147 this.id = id;
c48d2125f1e2 - Moved code for scanning character literals to separate function. aziz parents: 17 diff changeset	148 this.type = type;
c48d2125f1e2 - Moved code for scanning character literals to separate function. aziz parents: 17 diff changeset	149 this.loc = loc;
c48d2125f1e2 - Moved code for scanning character literals to separate function. aziz parents: 17 diff changeset	150 }
c48d2125f1e2 - Moved code for scanning character literals to separate function. aziz parents: 17 diff changeset	151 }
c48d2125f1e2 - Moved code for scanning character literals to separate function. aziz parents: 17 diff changeset	152
0 8ba2570de175 Initial import. aziz parents: diff changeset	153 class Lexer
8ba2570de175 Initial import. aziz parents: diff changeset	154 {
4 92df59b1ec4a - Started implementation of scan(). aziz parents: 3 diff changeset	155 Token token;
30 426767b94635 - Added code for parsing the '#line' special token. aziz parents: 29 diff changeset	156 string text;
4 92df59b1ec4a - Started implementation of scan(). aziz parents: 3 diff changeset	157 char* p;
92df59b1ec4a - Started implementation of scan(). aziz parents: 3 diff changeset	158 char* end;
92df59b1ec4a - Started implementation of scan(). aziz parents: 3 diff changeset	159
17 9bd0bac79479 - Removed Whitespace from enum list. aziz parents: 16 diff changeset	160 uint loc = 1; /// line of code
9bd0bac79479 - Removed Whitespace from enum list. aziz parents: 16 diff changeset	161
30 426767b94635 - Added code for parsing the '#line' special token. aziz parents: 29 diff changeset	162 char[] fileName;
426767b94635 - Added code for parsing the '#line' special token. aziz parents: 29 diff changeset	163
18 c48d2125f1e2 - Moved code for scanning character literals to separate function. aziz parents: 17 diff changeset	164 Problem[] errors;
c48d2125f1e2 - Moved code for scanning character literals to separate function. aziz parents: 17 diff changeset	165
28 3a9daccf7d96 - Added table for identifiers to Lexer. aziz parents: 27 diff changeset	166 Identifier[string] idtable;
3a9daccf7d96 - Added table for identifiers to Lexer. aziz parents: 27 diff changeset	167
30 426767b94635 - Added code for parsing the '#line' special token. aziz parents: 29 diff changeset	168 this(string text, string fileName)
4 92df59b1ec4a - Started implementation of scan(). aziz parents: 3 diff changeset	169 {
30 426767b94635 - Added code for parsing the '#line' special token. aziz parents: 29 diff changeset	170 this.fileName = fileName;
426767b94635 - Added code for parsing the '#line' special token. aziz parents: 29 diff changeset	171
4 92df59b1ec4a - Started implementation of scan(). aziz parents: 3 diff changeset	172 this.text = text;
92df59b1ec4a - Started implementation of scan(). aziz parents: 3 diff changeset	173 this.text.length = this.text.length + 1;
92df59b1ec4a - Started implementation of scan(). aziz parents: 3 diff changeset	174 this.text[$-1] = 0;
92df59b1ec4a - Started implementation of scan(). aziz parents: 3 diff changeset	175
92df59b1ec4a - Started implementation of scan(). aziz parents: 3 diff changeset	176 this.p = this.text.ptr;
92df59b1ec4a - Started implementation of scan(). aziz parents: 3 diff changeset	177 this.end = this.p + this.text.length;
28 3a9daccf7d96 - Added table for identifiers to Lexer. aziz parents: 27 diff changeset	178
3a9daccf7d96 - Added table for identifiers to Lexer. aziz parents: 27 diff changeset	179 loadKeywords();
4 92df59b1ec4a - Started implementation of scan(). aziz parents: 3 diff changeset	180 }
92df59b1ec4a - Started implementation of scan(). aziz parents: 3 diff changeset	181
92df59b1ec4a - Started implementation of scan(). aziz parents: 3 diff changeset	182 public void scan(out Token t)
3 4bbce78bfb1e - Added TOK enum. aziz parents: 2 diff changeset	183 {
4 92df59b1ec4a - Started implementation of scan(). aziz parents: 3 diff changeset	184 assert(p < end);
0 8ba2570de175 Initial import. aziz parents: diff changeset	185
10 3ee65d6e39c9 - Parsing // comments now. aziz parents: 9 diff changeset	186 uint c = *p;
4 92df59b1ec4a - Started implementation of scan(). aziz parents: 3 diff changeset	187
5 79b4e8848794 - Started writing XML generator. aziz parents: 4 diff changeset	188 while(1)
4 92df59b1ec4a - Started implementation of scan(). aziz parents: 3 diff changeset	189 {
5 79b4e8848794 - Started writing XML generator. aziz parents: 4 diff changeset	190 t.start = p;
16 476e8e55c1d4 - Added Whitespace to the character properties table. aziz parents: 15 diff changeset	191
5 79b4e8848794 - Started writing XML generator. aziz parents: 4 diff changeset	192 if (c == 0)
79b4e8848794 - Started writing XML generator. aziz parents: 4 diff changeset	193 {
17 9bd0bac79479 - Removed Whitespace from enum list. aziz parents: 16 diff changeset	194 ++p;
5 79b4e8848794 - Started writing XML generator. aziz parents: 4 diff changeset	195 t.type = TOK.EOF;
17 9bd0bac79479 - Removed Whitespace from enum list. aziz parents: 16 diff changeset	196 t.end = p;
5 79b4e8848794 - Started writing XML generator. aziz parents: 4 diff changeset	197 return;
79b4e8848794 - Started writing XML generator. aziz parents: 4 diff changeset	198 }
4 92df59b1ec4a - Started implementation of scan(). aziz parents: 3 diff changeset	199
17 9bd0bac79479 - Removed Whitespace from enum list. aziz parents: 16 diff changeset	200 if (c == '\n')
9bd0bac79479 - Removed Whitespace from enum list. aziz parents: 16 diff changeset	201 {
9bd0bac79479 - Removed Whitespace from enum list. aziz parents: 16 diff changeset	202 c = *++p;
9bd0bac79479 - Removed Whitespace from enum list. aziz parents: 16 diff changeset	203 ++loc;
9bd0bac79479 - Removed Whitespace from enum list. aziz parents: 16 diff changeset	204 continue;
9bd0bac79479 - Removed Whitespace from enum list. aziz parents: 16 diff changeset	205 }
9bd0bac79479 - Removed Whitespace from enum list. aziz parents: 16 diff changeset	206 else if (c == '\r')
9bd0bac79479 - Removed Whitespace from enum list. aziz parents: 16 diff changeset	207 {
9bd0bac79479 - Removed Whitespace from enum list. aziz parents: 16 diff changeset	208 c = *++p;
9bd0bac79479 - Removed Whitespace from enum list. aziz parents: 16 diff changeset	209 if (c != '\n')
9bd0bac79479 - Removed Whitespace from enum list. aziz parents: 16 diff changeset	210 ++loc;
9bd0bac79479 - Removed Whitespace from enum list. aziz parents: 16 diff changeset	211 continue;
9bd0bac79479 - Removed Whitespace from enum list. aziz parents: 16 diff changeset	212 }
32 d7011daa4740 - Added missing commas after the items in the messages table. aziz parents: 31 diff changeset	213 else if (c == LS[0] && p[1] == LS[1] && (p[2] == LS[2] \|\| p[2] == PS[2]))
d7011daa4740 - Added missing commas after the items in the messages table. aziz parents: 31 diff changeset	214 {
d7011daa4740 - Added missing commas after the items in the messages table. aziz parents: 31 diff changeset	215 p += 3;
d7011daa4740 - Added missing commas after the items in the messages table. aziz parents: 31 diff changeset	216 c = *p;
d7011daa4740 - Added missing commas after the items in the messages table. aziz parents: 31 diff changeset	217 continue;
d7011daa4740 - Added missing commas after the items in the messages table. aziz parents: 31 diff changeset	218 }
17 9bd0bac79479 - Removed Whitespace from enum list. aziz parents: 16 diff changeset	219
13 e5211758b63c - Added isidbeg() function. aziz parents: 12 diff changeset	220 if (isidbeg(c))
5 79b4e8848794 - Started writing XML generator. aziz parents: 4 diff changeset	221 {
33 cf3047cf3cd2 - Added code for parsing back quote and raw strings. aziz parents: 32 diff changeset	222 if (c == 'r' && p[1] == '"' && ++p)
31 94f09f4e988e - Added struct for strings to Token with 'pf' = postfix. aziz parents: 30 diff changeset	223 return scanRawStringLiteral(t);
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix. aziz parents: 30 diff changeset	224 if (c == 'x' && p[1] == '"')
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix. aziz parents: 30 diff changeset	225 return scanHexStringLiteral(t);
12 0989206cf73c - Added code to decode Unicode characters in identifiers. aziz parents: 11 diff changeset	226 Lidentifier:
5 79b4e8848794 - Started writing XML generator. aziz parents: 4 diff changeset	227 do
79b4e8848794 - Started writing XML generator. aziz parents: 4 diff changeset	228 { c = *++p; }
12 0989206cf73c - Added code to decode Unicode characters in identifiers. aziz parents: 11 diff changeset	229 while (isident(c) \|\| c & 128 && isUniAlpha(decodeUTF()))
28 3a9daccf7d96 - Added table for identifiers to Lexer. aziz parents: 27 diff changeset	230
5 79b4e8848794 - Started writing XML generator. aziz parents: 4 diff changeset	231 t.end = p;
28 3a9daccf7d96 - Added table for identifiers to Lexer. aziz parents: 27 diff changeset	232
3a9daccf7d96 - Added table for identifiers to Lexer. aziz parents: 27 diff changeset	233 string str = t.span;
3a9daccf7d96 - Added table for identifiers to Lexer. aziz parents: 27 diff changeset	234 Identifier* id = str in idtable;
3a9daccf7d96 - Added table for identifiers to Lexer. aziz parents: 27 diff changeset	235
3a9daccf7d96 - Added table for identifiers to Lexer. aziz parents: 27 diff changeset	236 if (!id)
3a9daccf7d96 - Added table for identifiers to Lexer. aziz parents: 27 diff changeset	237 {
3a9daccf7d96 - Added table for identifiers to Lexer. aziz parents: 27 diff changeset	238 idtable[str] = Identifier.Identifier(TOK.Identifier, str);
3a9daccf7d96 - Added table for identifiers to Lexer. aziz parents: 27 diff changeset	239 id = str in idtable;
3a9daccf7d96 - Added table for identifiers to Lexer. aziz parents: 27 diff changeset	240 }
3a9daccf7d96 - Added table for identifiers to Lexer. aziz parents: 27 diff changeset	241 assert(id);
3a9daccf7d96 - Added table for identifiers to Lexer. aziz parents: 27 diff changeset	242 t.type = id.type;
5 79b4e8848794 - Started writing XML generator. aziz parents: 4 diff changeset	243 return;
79b4e8848794 - Started writing XML generator. aziz parents: 4 diff changeset	244 }
79b4e8848794 - Started writing XML generator. aziz parents: 4 diff changeset	245
15 c70c028e47dd - Started implementation of lexing numbers. aziz parents: 14 diff changeset	246 if (isdigit(c))
c70c028e47dd - Started implementation of lexing numbers. aziz parents: 14 diff changeset	247 return scanNumber(t);
c70c028e47dd - Started implementation of lexing numbers. aziz parents: 14 diff changeset	248
8 d4ba94a5a282 - Parsing /* / comments now. aziz* parents: 7 diff changeset	249 if (c == '/')
5 79b4e8848794 - Started writing XML generator. aziz parents: 4 diff changeset	250 {
8 d4ba94a5a282 - Parsing /* / comments now. aziz* parents: 7 diff changeset	251 c = *++p;
14 cdf788d8bdaf - Parsing /= now. aziz parents: 13 diff changeset	252 switch(c)
5 79b4e8848794 - Started writing XML generator. aziz parents: 4 diff changeset	253 {
18 c48d2125f1e2 - Moved code for scanning character literals to separate function. aziz parents: 17 diff changeset	254 case '=':
14 cdf788d8bdaf - Parsing /= now. aziz parents: 13 diff changeset	255 ++p;
32 d7011daa4740 - Added missing commas after the items in the messages table. aziz parents: 31 diff changeset	256 t.type = TOK.DivAssign;
14 cdf788d8bdaf - Parsing /= now. aziz parents: 13 diff changeset	257 t.end = p;
cdf788d8bdaf - Parsing /= now. aziz parents: 13 diff changeset	258 return;
18 c48d2125f1e2 - Moved code for scanning character literals to separate function. aziz parents: 17 diff changeset	259 case '+':
8 d4ba94a5a282 - Parsing /* / comments now. aziz* parents: 7 diff changeset	260 uint level = 1;
32 d7011daa4740 - Added missing commas after the items in the messages table. aziz parents: 31 diff changeset	261 while (1)
d7011daa4740 - Added missing commas after the items in the messages table. aziz parents: 31 diff changeset	262 {
d7011daa4740 - Added missing commas after the items in the messages table. aziz parents: 31 diff changeset	263 c = *++p;
d7011daa4740 - Added missing commas after the items in the messages table. aziz parents: 31 diff changeset	264 switch (c)
d7011daa4740 - Added missing commas after the items in the messages table. aziz parents: 31 diff changeset	265 {
d7011daa4740 - Added missing commas after the items in the messages table. aziz parents: 31 diff changeset	266 case '\r':
d7011daa4740 - Added missing commas after the items in the messages table. aziz parents: 31 diff changeset	267 if (p[1] == '\n')
d7011daa4740 - Added missing commas after the items in the messages table. aziz parents: 31 diff changeset	268 ++p;
d7011daa4740 - Added missing commas after the items in the messages table. aziz parents: 31 diff changeset	269 case '\n':
d7011daa4740 - Added missing commas after the items in the messages table. aziz parents: 31 diff changeset	270 ++loc;
d7011daa4740 - Added missing commas after the items in the messages table. aziz parents: 31 diff changeset	271 continue;
d7011daa4740 - Added missing commas after the items in the messages table. aziz parents: 31 diff changeset	272 case '/':
d7011daa4740 - Added missing commas after the items in the messages table. aziz parents: 31 diff changeset	273 if (p[1] == '+')
d7011daa4740 - Added missing commas after the items in the messages table. aziz parents: 31 diff changeset	274 {
d7011daa4740 - Added missing commas after the items in the messages table. aziz parents: 31 diff changeset	275 ++p;
d7011daa4740 - Added missing commas after the items in the messages table. aziz parents: 31 diff changeset	276 ++level;
d7011daa4740 - Added missing commas after the items in the messages table. aziz parents: 31 diff changeset	277 }
d7011daa4740 - Added missing commas after the items in the messages table. aziz parents: 31 diff changeset	278 continue;
d7011daa4740 - Added missing commas after the items in the messages table. aziz parents: 31 diff changeset	279 case '+':
d7011daa4740 - Added missing commas after the items in the messages table. aziz parents: 31 diff changeset	280 if (p[1] == '/')
d7011daa4740 - Added missing commas after the items in the messages table. aziz parents: 31 diff changeset	281 {
d7011daa4740 - Added missing commas after the items in the messages table. aziz parents: 31 diff changeset	282 ++p;
d7011daa4740 - Added missing commas after the items in the messages table. aziz parents: 31 diff changeset	283 if (--level == 0)
d7011daa4740 - Added missing commas after the items in the messages table. aziz parents: 31 diff changeset	284 {
d7011daa4740 - Added missing commas after the items in the messages table. aziz parents: 31 diff changeset	285 ++p;
d7011daa4740 - Added missing commas after the items in the messages table. aziz parents: 31 diff changeset	286 LreturnNC:
d7011daa4740 - Added missing commas after the items in the messages table. aziz parents: 31 diff changeset	287 t.type = TOK.Comment;
d7011daa4740 - Added missing commas after the items in the messages table. aziz parents: 31 diff changeset	288 t.end = p;
d7011daa4740 - Added missing commas after the items in the messages table. aziz parents: 31 diff changeset	289 return;
d7011daa4740 - Added missing commas after the items in the messages table. aziz parents: 31 diff changeset	290 }
d7011daa4740 - Added missing commas after the items in the messages table. aziz parents: 31 diff changeset	291 }
d7011daa4740 - Added missing commas after the items in the messages table. aziz parents: 31 diff changeset	292 continue;
d7011daa4740 - Added missing commas after the items in the messages table. aziz parents: 31 diff changeset	293 case 0, _Z_:
d7011daa4740 - Added missing commas after the items in the messages table. aziz parents: 31 diff changeset	294 error(MID.UnterminatedNestedComment);
d7011daa4740 - Added missing commas after the items in the messages table. aziz parents: 31 diff changeset	295 goto LreturnNC;
d7011daa4740 - Added missing commas after the items in the messages table. aziz parents: 31 diff changeset	296 case LS[0]:
d7011daa4740 - Added missing commas after the items in the messages table. aziz parents: 31 diff changeset	297 if (p[1] == LS[1] && (p[2] == LS[2] \|\| p[2] == PS[2])) {
d7011daa4740 - Added missing commas after the items in the messages table. aziz parents: 31 diff changeset	298 p += 2;
d7011daa4740 - Added missing commas after the items in the messages table. aziz parents: 31 diff changeset	299 ++loc;
d7011daa4740 - Added missing commas after the items in the messages table. aziz parents: 31 diff changeset	300 }
d7011daa4740 - Added missing commas after the items in the messages table. aziz parents: 31 diff changeset	301 continue;
d7011daa4740 - Added missing commas after the items in the messages table. aziz parents: 31 diff changeset	302 default:
d7011daa4740 - Added missing commas after the items in the messages table. aziz parents: 31 diff changeset	303 }
d7011daa4740 - Added missing commas after the items in the messages table. aziz parents: 31 diff changeset	304 }
d7011daa4740 - Added missing commas after the items in the messages table. aziz parents: 31 diff changeset	305 case '*':
d7011daa4740 - Added missing commas after the items in the messages table. aziz parents: 31 diff changeset	306 while (1)
7 07e45c06a024 - Parsing nested comments correctly now. aziz parents: 5 diff changeset	307 {
8 d4ba94a5a282 - Parsing /* / comments now. aziz* parents: 7 diff changeset	308 c = *++p;
32 d7011daa4740 - Added missing commas after the items in the messages table. aziz parents: 31 diff changeset	309 switch (c)
8 d4ba94a5a282 - Parsing /* / comments now. aziz* parents: 7 diff changeset	310 {
32 d7011daa4740 - Added missing commas after the items in the messages table. aziz parents: 31 diff changeset	311 case '\r':
d7011daa4740 - Added missing commas after the items in the messages table. aziz parents: 31 diff changeset	312 if (p[1] == '\n')
d7011daa4740 - Added missing commas after the items in the messages table. aziz parents: 31 diff changeset	313 ++p;
d7011daa4740 - Added missing commas after the items in the messages table. aziz parents: 31 diff changeset	314 case '\n':
d7011daa4740 - Added missing commas after the items in the messages table. aziz parents: 31 diff changeset	315 ++loc;
d7011daa4740 - Added missing commas after the items in the messages table. aziz parents: 31 diff changeset	316 continue;
d7011daa4740 - Added missing commas after the items in the messages table. aziz parents: 31 diff changeset	317 case '*':
d7011daa4740 - Added missing commas after the items in the messages table. aziz parents: 31 diff changeset	318 if (p[1] == '/')
d7011daa4740 - Added missing commas after the items in the messages table. aziz parents: 31 diff changeset	319 {
d7011daa4740 - Added missing commas after the items in the messages table. aziz parents: 31 diff changeset	320 p += 2;
d7011daa4740 - Added missing commas after the items in the messages table. aziz parents: 31 diff changeset	321 LreturnBC:
d7011daa4740 - Added missing commas after the items in the messages table. aziz parents: 31 diff changeset	322 t.type = TOK.Comment;
d7011daa4740 - Added missing commas after the items in the messages table. aziz parents: 31 diff changeset	323 t.end = p;
d7011daa4740 - Added missing commas after the items in the messages table. aziz parents: 31 diff changeset	324 return;
d7011daa4740 - Added missing commas after the items in the messages table. aziz parents: 31 diff changeset	325 }
d7011daa4740 - Added missing commas after the items in the messages table. aziz parents: 31 diff changeset	326 break;
d7011daa4740 - Added missing commas after the items in the messages table. aziz parents: 31 diff changeset	327 case LS[0]:
d7011daa4740 - Added missing commas after the items in the messages table. aziz parents: 31 diff changeset	328 if (p[1] == LS[1] && (p[2] == LS[2] \|\| p[2] == PS[2])) {
d7011daa4740 - Added missing commas after the items in the messages table. aziz parents: 31 diff changeset	329 p += 2;
d7011daa4740 - Added missing commas after the items in the messages table. aziz parents: 31 diff changeset	330 ++loc;
d7011daa4740 - Added missing commas after the items in the messages table. aziz parents: 31 diff changeset	331 }
d7011daa4740 - Added missing commas after the items in the messages table. aziz parents: 31 diff changeset	332 break;
d7011daa4740 - Added missing commas after the items in the messages table. aziz parents: 31 diff changeset	333 case 0, _Z_:
d7011daa4740 - Added missing commas after the items in the messages table. aziz parents: 31 diff changeset	334 error(MID.UnterminatedBlockComment);
d7011daa4740 - Added missing commas after the items in the messages table. aziz parents: 31 diff changeset	335 goto LreturnBC;
d7011daa4740 - Added missing commas after the items in the messages table. aziz parents: 31 diff changeset	336 default:
8 d4ba94a5a282 - Parsing /* / comments now. aziz* parents: 7 diff changeset	337 }
32 d7011daa4740 - Added missing commas after the items in the messages table. aziz parents: 31 diff changeset	338 }
d7011daa4740 - Added missing commas after the items in the messages table. aziz parents: 31 diff changeset	339 assert(0);
d7011daa4740 - Added missing commas after the items in the messages table. aziz parents: 31 diff changeset	340 case '/':
d7011daa4740 - Added missing commas after the items in the messages table. aziz parents: 31 diff changeset	341 while (1)
7 07e45c06a024 - Parsing nested comments correctly now. aziz parents: 5 diff changeset	342 {
8 d4ba94a5a282 - Parsing /* / comments now. aziz* parents: 7 diff changeset	343 c = *++p;
32 d7011daa4740 - Added missing commas after the items in the messages table. aziz parents: 31 diff changeset	344 switch (c)
d7011daa4740 - Added missing commas after the items in the messages table. aziz parents: 31 diff changeset	345 {
d7011daa4740 - Added missing commas after the items in the messages table. aziz parents: 31 diff changeset	346 case '\r':
d7011daa4740 - Added missing commas after the items in the messages table. aziz parents: 31 diff changeset	347 if (p[1] == '\n')
d7011daa4740 - Added missing commas after the items in the messages table. aziz parents: 31 diff changeset	348 ++p;
d7011daa4740 - Added missing commas after the items in the messages table. aziz parents: 31 diff changeset	349 case '\n':
d7011daa4740 - Added missing commas after the items in the messages table. aziz parents: 31 diff changeset	350 case 0, _Z_:
11 dffcdaa7c47a - Added Unicode line and paragraph separators. aziz parents: 10 diff changeset	351 break;
32 d7011daa4740 - Added missing commas after the items in the messages table. aziz parents: 31 diff changeset	352 case LS[0]:
d7011daa4740 - Added missing commas after the items in the messages table. aziz parents: 31 diff changeset	353 if (p[1] == LS[1] && (p[2] == LS[2] \|\| p[2] == PS[2]))
d7011daa4740 - Added missing commas after the items in the messages table. aziz parents: 31 diff changeset	354 break;
d7011daa4740 - Added missing commas after the items in the messages table. aziz parents: 31 diff changeset	355 continue;
d7011daa4740 - Added missing commas after the items in the messages table. aziz parents: 31 diff changeset	356 default:
d7011daa4740 - Added missing commas after the items in the messages table. aziz parents: 31 diff changeset	357 continue;
d7011daa4740 - Added missing commas after the items in the messages table. aziz parents: 31 diff changeset	358 }
d7011daa4740 - Added missing commas after the items in the messages table. aziz parents: 31 diff changeset	359 t.type = TOK.Comment;
d7011daa4740 - Added missing commas after the items in the messages table. aziz parents: 31 diff changeset	360 t.end = p;
d7011daa4740 - Added missing commas after the items in the messages table. aziz parents: 31 diff changeset	361 return;
d7011daa4740 - Added missing commas after the items in the messages table. aziz parents: 31 diff changeset	362 }
d7011daa4740 - Added missing commas after the items in the messages table. aziz parents: 31 diff changeset	363 default:
d7011daa4740 - Added missing commas after the items in the messages table. aziz parents: 31 diff changeset	364 t.type = TOK.Div;
10 3ee65d6e39c9 - Parsing // comments now. aziz parents: 9 diff changeset	365 t.end = p;
3ee65d6e39c9 - Parsing // comments now. aziz parents: 9 diff changeset	366 return;
3ee65d6e39c9 - Parsing // comments now. aziz parents: 9 diff changeset	367 }
5 79b4e8848794 - Started writing XML generator. aziz parents: 4 diff changeset	368 }
79b4e8848794 - Started writing XML generator. aziz parents: 4 diff changeset	369
9 5d6968cc751e - Parsing string and character literals now (rudimentary implementation.) aziz parents: 8 diff changeset	370 if (c == '"')
5d6968cc751e - Parsing string and character literals now (rudimentary implementation.) aziz parents: 8 diff changeset	371 {
5d6968cc751e - Parsing string and character literals now (rudimentary implementation.) aziz parents: 8 diff changeset	372 do {
5d6968cc751e - Parsing string and character literals now (rudimentary implementation.) aziz parents: 8 diff changeset	373 c = *++p;
5d6968cc751e - Parsing string and character literals now (rudimentary implementation.) aziz parents: 8 diff changeset	374 if (c == 0)
5d6968cc751e - Parsing string and character literals now (rudimentary implementation.) aziz parents: 8 diff changeset	375 throw new Error("unterminated string literal.");
5d6968cc751e - Parsing string and character literals now (rudimentary implementation.) aziz parents: 8 diff changeset	376 if (c == '\\')
5d6968cc751e - Parsing string and character literals now (rudimentary implementation.) aziz parents: 8 diff changeset	377 ++p;
5d6968cc751e - Parsing string and character literals now (rudimentary implementation.) aziz parents: 8 diff changeset	378 } while (c != '"')
5d6968cc751e - Parsing string and character literals now (rudimentary implementation.) aziz parents: 8 diff changeset	379 ++p;
5d6968cc751e - Parsing string and character literals now (rudimentary implementation.) aziz parents: 8 diff changeset	380 t.type = TOK.String;
5d6968cc751e - Parsing string and character literals now (rudimentary implementation.) aziz parents: 8 diff changeset	381 t.end = p;
5d6968cc751e - Parsing string and character literals now (rudimentary implementation.) aziz parents: 8 diff changeset	382 return;
5d6968cc751e - Parsing string and character literals now (rudimentary implementation.) aziz parents: 8 diff changeset	383 }
5d6968cc751e - Parsing string and character literals now (rudimentary implementation.) aziz parents: 8 diff changeset	384
5d6968cc751e - Parsing string and character literals now (rudimentary implementation.) aziz parents: 8 diff changeset	385 if (c == '\'')
18 c48d2125f1e2 - Moved code for scanning character literals to separate function. aziz parents: 17 diff changeset	386 return scanCharacterLiteral(t);
12 0989206cf73c - Added code to decode Unicode characters in identifiers. aziz parents: 11 diff changeset	387
33 cf3047cf3cd2 - Added code for parsing back quote and raw strings. aziz parents: 32 diff changeset	388 if (c == '`')
cf3047cf3cd2 - Added code for parsing back quote and raw strings. aziz parents: 32 diff changeset	389 return scanRawStringLiteral(t);
cf3047cf3cd2 - Added code for parsing back quote and raw strings. aziz parents: 32 diff changeset	390
20 d6adfbd7c513 - Added code for parsing braces. aziz parents: 19 diff changeset	391 switch(c)
d6adfbd7c513 - Added code for parsing braces. aziz parents: 19 diff changeset	392 {
22 b05fff8e2ce4 - Added code for parsing Dot, Slice and Ellipses tokens. aziz parents: 21 diff changeset	393 case '.':
b05fff8e2ce4 - Added code for parsing Dot, Slice and Ellipses tokens. aziz parents: 21 diff changeset	394 if (p[1] == '.')
b05fff8e2ce4 - Added code for parsing Dot, Slice and Ellipses tokens. aziz parents: 21 diff changeset	395 {
b05fff8e2ce4 - Added code for parsing Dot, Slice and Ellipses tokens. aziz parents: 21 diff changeset	396 ++p;
b05fff8e2ce4 - Added code for parsing Dot, Slice and Ellipses tokens. aziz parents: 21 diff changeset	397 if (p[1] == '.') {
b05fff8e2ce4 - Added code for parsing Dot, Slice and Ellipses tokens. aziz parents: 21 diff changeset	398 ++p;
b05fff8e2ce4 - Added code for parsing Dot, Slice and Ellipses tokens. aziz parents: 21 diff changeset	399 t.type = TOK.Ellipses;
b05fff8e2ce4 - Added code for parsing Dot, Slice and Ellipses tokens. aziz parents: 21 diff changeset	400 }
b05fff8e2ce4 - Added code for parsing Dot, Slice and Ellipses tokens. aziz parents: 21 diff changeset	401 else
b05fff8e2ce4 - Added code for parsing Dot, Slice and Ellipses tokens. aziz parents: 21 diff changeset	402 t.type = TOK.Slice;
b05fff8e2ce4 - Added code for parsing Dot, Slice and Ellipses tokens. aziz parents: 21 diff changeset	403 }
b05fff8e2ce4 - Added code for parsing Dot, Slice and Ellipses tokens. aziz parents: 21 diff changeset	404 else
b05fff8e2ce4 - Added code for parsing Dot, Slice and Ellipses tokens. aziz parents: 21 diff changeset	405 t.type = TOK.Dot;
b05fff8e2ce4 - Added code for parsing Dot, Slice and Ellipses tokens. aziz parents: 21 diff changeset	406 goto Lcommon;
23 1a7903701a3d - Added code for parsing OrAssign, OrLogical and OrBinary tokens. aziz parents: 22 diff changeset	407 case '\|':
1a7903701a3d - Added code for parsing OrAssign, OrLogical and OrBinary tokens. aziz parents: 22 diff changeset	408 c = *++p;
1a7903701a3d - Added code for parsing OrAssign, OrLogical and OrBinary tokens. aziz parents: 22 diff changeset	409 if (c == '=')
1a7903701a3d - Added code for parsing OrAssign, OrLogical and OrBinary tokens. aziz parents: 22 diff changeset	410 t.type = TOK.OrAssign;
1a7903701a3d - Added code for parsing OrAssign, OrLogical and OrBinary tokens. aziz parents: 22 diff changeset	411 else if (c == '\|')
1a7903701a3d - Added code for parsing OrAssign, OrLogical and OrBinary tokens. aziz parents: 22 diff changeset	412 t.type = TOK.OrLogical;
26 c3d7373db241 - Added code for parsing Assign and Equal tokens. aziz parents: 25 diff changeset	413 else {
23 1a7903701a3d - Added code for parsing OrAssign, OrLogical and OrBinary tokens. aziz parents: 22 diff changeset	414 t.type = TOK.OrBinary;
26 c3d7373db241 - Added code for parsing Assign and Equal tokens. aziz parents: 25 diff changeset	415 goto Lcommon2;
c3d7373db241 - Added code for parsing Assign and Equal tokens. aziz parents: 25 diff changeset	416 }
23 1a7903701a3d - Added code for parsing OrAssign, OrLogical and OrBinary tokens. aziz parents: 22 diff changeset	417 goto Lcommon;
24 903f91163f23 - Added code for parsing AndAssign, AndLogical and AndBinary tokens. aziz parents: 23 diff changeset	418 case '&':
903f91163f23 - Added code for parsing AndAssign, AndLogical and AndBinary tokens. aziz parents: 23 diff changeset	419 c = *++p;
903f91163f23 - Added code for parsing AndAssign, AndLogical and AndBinary tokens. aziz parents: 23 diff changeset	420 if (c == '=')
903f91163f23 - Added code for parsing AndAssign, AndLogical and AndBinary tokens. aziz parents: 23 diff changeset	421 t.type = TOK.AndAssign;
903f91163f23 - Added code for parsing AndAssign, AndLogical and AndBinary tokens. aziz parents: 23 diff changeset	422 else if (c == '&')
903f91163f23 - Added code for parsing AndAssign, AndLogical and AndBinary tokens. aziz parents: 23 diff changeset	423 t.type = TOK.AndLogical;
26 c3d7373db241 - Added code for parsing Assign and Equal tokens. aziz parents: 25 diff changeset	424 else {
24 903f91163f23 - Added code for parsing AndAssign, AndLogical and AndBinary tokens. aziz parents: 23 diff changeset	425 t.type = TOK.AndBinary;
26 c3d7373db241 - Added code for parsing Assign and Equal tokens. aziz parents: 25 diff changeset	426 goto Lcommon2;
c3d7373db241 - Added code for parsing Assign and Equal tokens. aziz parents: 25 diff changeset	427 }
24 903f91163f23 - Added code for parsing AndAssign, AndLogical and AndBinary tokens. aziz parents: 23 diff changeset	428 goto Lcommon;
25 9c866208b3f6 - Added code for parsing PlusAssign, PlusPlus, Plus, MinusAssign, MinusMinus, Minus tokens. aziz parents: 24 diff changeset	429 case '+':
9c866208b3f6 - Added code for parsing PlusAssign, PlusPlus, Plus, MinusAssign, MinusMinus, Minus tokens. aziz parents: 24 diff changeset	430 c = *++p;
9c866208b3f6 - Added code for parsing PlusAssign, PlusPlus, Plus, MinusAssign, MinusMinus, Minus tokens. aziz parents: 24 diff changeset	431 if (c == '=')
9c866208b3f6 - Added code for parsing PlusAssign, PlusPlus, Plus, MinusAssign, MinusMinus, Minus tokens. aziz parents: 24 diff changeset	432 t.type = TOK.PlusAssign;
9c866208b3f6 - Added code for parsing PlusAssign, PlusPlus, Plus, MinusAssign, MinusMinus, Minus tokens. aziz parents: 24 diff changeset	433 else if (c == '+')
9c866208b3f6 - Added code for parsing PlusAssign, PlusPlus, Plus, MinusAssign, MinusMinus, Minus tokens. aziz parents: 24 diff changeset	434 t.type = TOK.PlusPlus;
26 c3d7373db241 - Added code for parsing Assign and Equal tokens. aziz parents: 25 diff changeset	435 else {
25 9c866208b3f6 - Added code for parsing PlusAssign, PlusPlus, Plus, MinusAssign, MinusMinus, Minus tokens. aziz parents: 24 diff changeset	436 t.type = TOK.Plus;
26 c3d7373db241 - Added code for parsing Assign and Equal tokens. aziz parents: 25 diff changeset	437 goto Lcommon2;
c3d7373db241 - Added code for parsing Assign and Equal tokens. aziz parents: 25 diff changeset	438 }
25 9c866208b3f6 - Added code for parsing PlusAssign, PlusPlus, Plus, MinusAssign, MinusMinus, Minus tokens. aziz parents: 24 diff changeset	439 goto Lcommon;
9c866208b3f6 - Added code for parsing PlusAssign, PlusPlus, Plus, MinusAssign, MinusMinus, Minus tokens. aziz parents: 24 diff changeset	440 case '-':
9c866208b3f6 - Added code for parsing PlusAssign, PlusPlus, Plus, MinusAssign, MinusMinus, Minus tokens. aziz parents: 24 diff changeset	441 c = *++p;
9c866208b3f6 - Added code for parsing PlusAssign, PlusPlus, Plus, MinusAssign, MinusMinus, Minus tokens. aziz parents: 24 diff changeset	442 if (c == '=')
9c866208b3f6 - Added code for parsing PlusAssign, PlusPlus, Plus, MinusAssign, MinusMinus, Minus tokens. aziz parents: 24 diff changeset	443 t.type = TOK.MinusAssign;
9c866208b3f6 - Added code for parsing PlusAssign, PlusPlus, Plus, MinusAssign, MinusMinus, Minus tokens. aziz parents: 24 diff changeset	444 else if (c == '-')
9c866208b3f6 - Added code for parsing PlusAssign, PlusPlus, Plus, MinusAssign, MinusMinus, Minus tokens. aziz parents: 24 diff changeset	445 t.type = TOK.MinusMinus;
26 c3d7373db241 - Added code for parsing Assign and Equal tokens. aziz parents: 25 diff changeset	446 else {
c3d7373db241 - Added code for parsing Assign and Equal tokens. aziz parents: 25 diff changeset	447 t.type = TOK.Minus;
c3d7373db241 - Added code for parsing Assign and Equal tokens. aziz parents: 25 diff changeset	448 goto Lcommon2;
c3d7373db241 - Added code for parsing Assign and Equal tokens. aziz parents: 25 diff changeset	449 }
c3d7373db241 - Added code for parsing Assign and Equal tokens. aziz parents: 25 diff changeset	450 goto Lcommon;
c3d7373db241 - Added code for parsing Assign and Equal tokens. aziz parents: 25 diff changeset	451 case '=':
c3d7373db241 - Added code for parsing Assign and Equal tokens. aziz parents: 25 diff changeset	452 if (p[1] == '=') {
c3d7373db241 - Added code for parsing Assign and Equal tokens. aziz parents: 25 diff changeset	453 ++p;
c3d7373db241 - Added code for parsing Assign and Equal tokens. aziz parents: 25 diff changeset	454 t.type = TOK.Equal;
c3d7373db241 - Added code for parsing Assign and Equal tokens. aziz parents: 25 diff changeset	455 }
25 9c866208b3f6 - Added code for parsing PlusAssign, PlusPlus, Plus, MinusAssign, MinusMinus, Minus tokens. aziz parents: 24 diff changeset	456 else
26 c3d7373db241 - Added code for parsing Assign and Equal tokens. aziz parents: 25 diff changeset	457 t.type = TOK.Assign;
25 9c866208b3f6 - Added code for parsing PlusAssign, PlusPlus, Plus, MinusAssign, MinusMinus, Minus tokens. aziz parents: 24 diff changeset	458 goto Lcommon;
27 43b6bf56f0e9 - Added code for parsing CatAssign and Tilde tokens. aziz parents: 26 diff changeset	459 case '~':
43b6bf56f0e9 - Added code for parsing CatAssign and Tilde tokens. aziz parents: 26 diff changeset	460 if (p[1] == '=') {
43b6bf56f0e9 - Added code for parsing CatAssign and Tilde tokens. aziz parents: 26 diff changeset	461 ++p;
43b6bf56f0e9 - Added code for parsing CatAssign and Tilde tokens. aziz parents: 26 diff changeset	462 t.type = TOK.CatAssign;
43b6bf56f0e9 - Added code for parsing CatAssign and Tilde tokens. aziz parents: 26 diff changeset	463 }
43b6bf56f0e9 - Added code for parsing CatAssign and Tilde tokens. aziz parents: 26 diff changeset	464 else
43b6bf56f0e9 - Added code for parsing CatAssign and Tilde tokens. aziz parents: 26 diff changeset	465 t.type = TOK.Tilde;
43b6bf56f0e9 - Added code for parsing CatAssign and Tilde tokens. aziz parents: 26 diff changeset	466 goto Lcommon;
29 ef83eea26bbd - Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens. aziz parents: 28 diff changeset	467 case '*':
ef83eea26bbd - Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens. aziz parents: 28 diff changeset	468 if (p[1] == '=') {
ef83eea26bbd - Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens. aziz parents: 28 diff changeset	469 ++p;
ef83eea26bbd - Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens. aziz parents: 28 diff changeset	470 t.type = TOK.MulAssign;
ef83eea26bbd - Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens. aziz parents: 28 diff changeset	471 }
ef83eea26bbd - Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens. aziz parents: 28 diff changeset	472 else
ef83eea26bbd - Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens. aziz parents: 28 diff changeset	473 t.type = TOK.Mul;
ef83eea26bbd - Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens. aziz parents: 28 diff changeset	474 goto Lcommon;
ef83eea26bbd - Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens. aziz parents: 28 diff changeset	475 case '^':
ef83eea26bbd - Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens. aziz parents: 28 diff changeset	476 if (p[1] == '=') {
ef83eea26bbd - Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens. aziz parents: 28 diff changeset	477 ++p;
ef83eea26bbd - Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens. aziz parents: 28 diff changeset	478 t.type = TOK.XorAssign;
ef83eea26bbd - Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens. aziz parents: 28 diff changeset	479 }
ef83eea26bbd - Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens. aziz parents: 28 diff changeset	480 else
ef83eea26bbd - Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens. aziz parents: 28 diff changeset	481 t.type = TOK.Xor;
ef83eea26bbd - Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens. aziz parents: 28 diff changeset	482 goto Lcommon;
ef83eea26bbd - Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens. aziz parents: 28 diff changeset	483 case '%':
ef83eea26bbd - Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens. aziz parents: 28 diff changeset	484 if (p[1] == '=') {
ef83eea26bbd - Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens. aziz parents: 28 diff changeset	485 ++p;
ef83eea26bbd - Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens. aziz parents: 28 diff changeset	486 t.type = TOK.ModAssign;
ef83eea26bbd - Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens. aziz parents: 28 diff changeset	487 }
ef83eea26bbd - Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens. aziz parents: 28 diff changeset	488 else
ef83eea26bbd - Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens. aziz parents: 28 diff changeset	489 t.type = TOK.Mod;
ef83eea26bbd - Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens. aziz parents: 28 diff changeset	490 goto Lcommon;
ef83eea26bbd - Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens. aziz parents: 28 diff changeset	491 // Single character tokens:
20 d6adfbd7c513 - Added code for parsing braces. aziz parents: 19 diff changeset	492 case '(':
d6adfbd7c513 - Added code for parsing braces. aziz parents: 19 diff changeset	493 t.type = TOK.LParen;
d6adfbd7c513 - Added code for parsing braces. aziz parents: 19 diff changeset	494 goto Lcommon;
d6adfbd7c513 - Added code for parsing braces. aziz parents: 19 diff changeset	495 case ')':
d6adfbd7c513 - Added code for parsing braces. aziz parents: 19 diff changeset	496 t.type = TOK.RParen;
d6adfbd7c513 - Added code for parsing braces. aziz parents: 19 diff changeset	497 goto Lcommon;
d6adfbd7c513 - Added code for parsing braces. aziz parents: 19 diff changeset	498 case '[':
d6adfbd7c513 - Added code for parsing braces. aziz parents: 19 diff changeset	499 t.type = TOK.LBracket;
d6adfbd7c513 - Added code for parsing braces. aziz parents: 19 diff changeset	500 goto Lcommon;
d6adfbd7c513 - Added code for parsing braces. aziz parents: 19 diff changeset	501 case ']':
d6adfbd7c513 - Added code for parsing braces. aziz parents: 19 diff changeset	502 t.type = TOK.RBracket;
d6adfbd7c513 - Added code for parsing braces. aziz parents: 19 diff changeset	503 goto Lcommon;
d6adfbd7c513 - Added code for parsing braces. aziz parents: 19 diff changeset	504 case '{':
d6adfbd7c513 - Added code for parsing braces. aziz parents: 19 diff changeset	505 t.type = TOK.LBrace;
d6adfbd7c513 - Added code for parsing braces. aziz parents: 19 diff changeset	506 goto Lcommon;
d6adfbd7c513 - Added code for parsing braces. aziz parents: 19 diff changeset	507 case '}':
d6adfbd7c513 - Added code for parsing braces. aziz parents: 19 diff changeset	508 t.type = TOK.RBrace;
21 c785c122e4e6 - Added code for parsing Colon, Semicolon, Question, Comma and Dollar. aziz parents: 20 diff changeset	509 goto Lcommon;
c785c122e4e6 - Added code for parsing Colon, Semicolon, Question, Comma and Dollar. aziz parents: 20 diff changeset	510 case ':':
c785c122e4e6 - Added code for parsing Colon, Semicolon, Question, Comma and Dollar. aziz parents: 20 diff changeset	511 t.type = TOK.Colon;
c785c122e4e6 - Added code for parsing Colon, Semicolon, Question, Comma and Dollar. aziz parents: 20 diff changeset	512 goto Lcommon;
c785c122e4e6 - Added code for parsing Colon, Semicolon, Question, Comma and Dollar. aziz parents: 20 diff changeset	513 case ';':
c785c122e4e6 - Added code for parsing Colon, Semicolon, Question, Comma and Dollar. aziz parents: 20 diff changeset	514 t.type = TOK.Semicolon;
c785c122e4e6 - Added code for parsing Colon, Semicolon, Question, Comma and Dollar. aziz parents: 20 diff changeset	515 goto Lcommon;
c785c122e4e6 - Added code for parsing Colon, Semicolon, Question, Comma and Dollar. aziz parents: 20 diff changeset	516 case '?':
c785c122e4e6 - Added code for parsing Colon, Semicolon, Question, Comma and Dollar. aziz parents: 20 diff changeset	517 t.type = TOK.Question;
c785c122e4e6 - Added code for parsing Colon, Semicolon, Question, Comma and Dollar. aziz parents: 20 diff changeset	518 goto Lcommon;
c785c122e4e6 - Added code for parsing Colon, Semicolon, Question, Comma and Dollar. aziz parents: 20 diff changeset	519 case ',':
c785c122e4e6 - Added code for parsing Colon, Semicolon, Question, Comma and Dollar. aziz parents: 20 diff changeset	520 t.type = TOK.Comma;
c785c122e4e6 - Added code for parsing Colon, Semicolon, Question, Comma and Dollar. aziz parents: 20 diff changeset	521 goto Lcommon;
c785c122e4e6 - Added code for parsing Colon, Semicolon, Question, Comma and Dollar. aziz parents: 20 diff changeset	522 case '$':
c785c122e4e6 - Added code for parsing Colon, Semicolon, Question, Comma and Dollar. aziz parents: 20 diff changeset	523 t.type = TOK.Dollar;
20 d6adfbd7c513 - Added code for parsing braces. aziz parents: 19 diff changeset	524 Lcommon:
d6adfbd7c513 - Added code for parsing braces. aziz parents: 19 diff changeset	525 ++p;
26 c3d7373db241 - Added code for parsing Assign and Equal tokens. aziz parents: 25 diff changeset	526 Lcommon2:
20 d6adfbd7c513 - Added code for parsing braces. aziz parents: 19 diff changeset	527 t.end = p;
d6adfbd7c513 - Added code for parsing braces. aziz parents: 19 diff changeset	528 return;
30 426767b94635 - Added code for parsing the '#line' special token. aziz parents: 29 diff changeset	529 case '#':
426767b94635 - Added code for parsing the '#line' special token. aziz parents: 29 diff changeset	530 ++p;
426767b94635 - Added code for parsing the '#line' special token. aziz parents: 29 diff changeset	531 scanSpecialToken();
426767b94635 - Added code for parsing the '#line' special token. aziz parents: 29 diff changeset	532 break;
20 d6adfbd7c513 - Added code for parsing braces. aziz parents: 19 diff changeset	533 default:
d6adfbd7c513 - Added code for parsing braces. aziz parents: 19 diff changeset	534 }
d6adfbd7c513 - Added code for parsing braces. aziz parents: 19 diff changeset	535
12 0989206cf73c - Added code to decode Unicode characters in identifiers. aziz parents: 11 diff changeset	536 if (c & 128 && isUniAlpha(decodeUTF()))
0989206cf73c - Added code to decode Unicode characters in identifiers. aziz parents: 11 diff changeset	537 goto Lidentifier;
5 79b4e8848794 - Started writing XML generator. aziz parents: 4 diff changeset	538 c = *++p;
4 92df59b1ec4a - Started implementation of scan(). aziz parents: 3 diff changeset	539 }
3 4bbce78bfb1e - Added TOK enum. aziz parents: 2 diff changeset	540 }
4 92df59b1ec4a - Started implementation of scan(). aziz parents: 3 diff changeset	541
30 426767b94635 - Added code for parsing the '#line' special token. aziz parents: 29 diff changeset	542 void peek(ref Token t)
426767b94635 - Added code for parsing the '#line' special token. aziz parents: 29 diff changeset	543 {
426767b94635 - Added code for parsing the '#line' special token. aziz parents: 29 diff changeset	544 char* tmp = p;
426767b94635 - Added code for parsing the '#line' special token. aziz parents: 29 diff changeset	545 scan(t);
426767b94635 - Added code for parsing the '#line' special token. aziz parents: 29 diff changeset	546 p = tmp;
426767b94635 - Added code for parsing the '#line' special token. aziz parents: 29 diff changeset	547 }
426767b94635 - Added code for parsing the '#line' special token. aziz parents: 29 diff changeset	548
18 c48d2125f1e2 - Moved code for scanning character literals to separate function. aziz parents: 17 diff changeset	549 void scanCharacterLiteral(ref Token t)
c48d2125f1e2 - Moved code for scanning character literals to separate function. aziz parents: 17 diff changeset	550 {
c48d2125f1e2 - Moved code for scanning character literals to separate function. aziz parents: 17 diff changeset	551 assert(*p == '\'');
c48d2125f1e2 - Moved code for scanning character literals to separate function. aziz parents: 17 diff changeset	552 MID id = MID.UnterminatedCharacterLiteral;
c48d2125f1e2 - Moved code for scanning character literals to separate function. aziz parents: 17 diff changeset	553 uint c = *++p;
c48d2125f1e2 - Moved code for scanning character literals to separate function. aziz parents: 17 diff changeset	554 switch(c)
c48d2125f1e2 - Moved code for scanning character literals to separate function. aziz parents: 17 diff changeset	555 {
c48d2125f1e2 - Moved code for scanning character literals to separate function. aziz parents: 17 diff changeset	556 case '\\':
c48d2125f1e2 - Moved code for scanning character literals to separate function. aziz parents: 17 diff changeset	557 ++p;
c48d2125f1e2 - Moved code for scanning character literals to separate function. aziz parents: 17 diff changeset	558 break;
32 d7011daa4740 - Added missing commas after the items in the messages table. aziz parents: 31 diff changeset	559 case 0, _Z_, '\n', '\r':
18 c48d2125f1e2 - Moved code for scanning character literals to separate function. aziz parents: 17 diff changeset	560 goto Lerr;
c48d2125f1e2 - Moved code for scanning character literals to separate function. aziz parents: 17 diff changeset	561 case '\'':
c48d2125f1e2 - Moved code for scanning character literals to separate function. aziz parents: 17 diff changeset	562 id = MID.EmptyCharacterLiteral;
c48d2125f1e2 - Moved code for scanning character literals to separate function. aziz parents: 17 diff changeset	563 goto Lerr;
c48d2125f1e2 - Moved code for scanning character literals to separate function. aziz parents: 17 diff changeset	564 default:
c48d2125f1e2 - Moved code for scanning character literals to separate function. aziz parents: 17 diff changeset	565 if (c & 128)
c48d2125f1e2 - Moved code for scanning character literals to separate function. aziz parents: 17 diff changeset	566 {
c48d2125f1e2 - Moved code for scanning character literals to separate function. aziz parents: 17 diff changeset	567 c = decodeUTF();
c48d2125f1e2 - Moved code for scanning character literals to separate function. aziz parents: 17 diff changeset	568 if (c == LSd \|\| c == PSd)
c48d2125f1e2 - Moved code for scanning character literals to separate function. aziz parents: 17 diff changeset	569 goto Lerr;
c48d2125f1e2 - Moved code for scanning character literals to separate function. aziz parents: 17 diff changeset	570 t.chr = c;
c48d2125f1e2 - Moved code for scanning character literals to separate function. aziz parents: 17 diff changeset	571 }
c48d2125f1e2 - Moved code for scanning character literals to separate function. aziz parents: 17 diff changeset	572 }
c48d2125f1e2 - Moved code for scanning character literals to separate function. aziz parents: 17 diff changeset	573
19 f85832f9f24e - Parsing character literals more correctly. aziz parents: 18 diff changeset	574 ++p;
18 c48d2125f1e2 - Moved code for scanning character literals to separate function. aziz parents: 17 diff changeset	575 if (*p != '\'')
c48d2125f1e2 - Moved code for scanning character literals to separate function. aziz parents: 17 diff changeset	576 Lerr:
c48d2125f1e2 - Moved code for scanning character literals to separate function. aziz parents: 17 diff changeset	577 error(id);
c48d2125f1e2 - Moved code for scanning character literals to separate function. aziz parents: 17 diff changeset	578 ++p;
c48d2125f1e2 - Moved code for scanning character literals to separate function. aziz parents: 17 diff changeset	579 t.type = TOK.Character;
c48d2125f1e2 - Moved code for scanning character literals to separate function. aziz parents: 17 diff changeset	580 t.end = p;
c48d2125f1e2 - Moved code for scanning character literals to separate function. aziz parents: 17 diff changeset	581 }
c48d2125f1e2 - Moved code for scanning character literals to separate function. aziz parents: 17 diff changeset	582
33 cf3047cf3cd2 - Added code for parsing back quote and raw strings. aziz parents: 32 diff changeset	583 char scanPostfix()
31 94f09f4e988e - Added struct for strings to Token with 'pf' = postfix. aziz parents: 30 diff changeset	584 {
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix. aziz parents: 30 diff changeset	585 switch (*p)
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix. aziz parents: 30 diff changeset	586 {
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix. aziz parents: 30 diff changeset	587 case 'c':
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix. aziz parents: 30 diff changeset	588 case 'w':
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix. aziz parents: 30 diff changeset	589 case 'd':
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix. aziz parents: 30 diff changeset	590 return *p++;
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix. aziz parents: 30 diff changeset	591 default:
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix. aziz parents: 30 diff changeset	592 return 0;
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix. aziz parents: 30 diff changeset	593 }
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix. aziz parents: 30 diff changeset	594 }
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix. aziz parents: 30 diff changeset	595
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix. aziz parents: 30 diff changeset	596 void scanRawStringLiteral(ref Token t)
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix. aziz parents: 30 diff changeset	597 {
33 cf3047cf3cd2 - Added code for parsing back quote and raw strings. aziz parents: 32 diff changeset	598 uint delim = *p;
cf3047cf3cd2 - Added code for parsing back quote and raw strings. aziz parents: 32 diff changeset	599 assert(delim == '`' \|\| delim == '"' && p[-1] == 'r');
cf3047cf3cd2 - Added code for parsing back quote and raw strings. aziz parents: 32 diff changeset	600 t.type = TOK.String;
cf3047cf3cd2 - Added code for parsing back quote and raw strings. aziz parents: 32 diff changeset	601 char[] buffer;
cf3047cf3cd2 - Added code for parsing back quote and raw strings. aziz parents: 32 diff changeset	602 uint c;
cf3047cf3cd2 - Added code for parsing back quote and raw strings. aziz parents: 32 diff changeset	603 while (1)
cf3047cf3cd2 - Added code for parsing back quote and raw strings. aziz parents: 32 diff changeset	604 {
cf3047cf3cd2 - Added code for parsing back quote and raw strings. aziz parents: 32 diff changeset	605 c = *++p;
cf3047cf3cd2 - Added code for parsing back quote and raw strings. aziz parents: 32 diff changeset	606 switch (c)
cf3047cf3cd2 - Added code for parsing back quote and raw strings. aziz parents: 32 diff changeset	607 {
cf3047cf3cd2 - Added code for parsing back quote and raw strings. aziz parents: 32 diff changeset	608 case '\r':
cf3047cf3cd2 - Added code for parsing back quote and raw strings. aziz parents: 32 diff changeset	609 if (p[1] == '\n')
cf3047cf3cd2 - Added code for parsing back quote and raw strings. aziz parents: 32 diff changeset	610 ++p;
cf3047cf3cd2 - Added code for parsing back quote and raw strings. aziz parents: 32 diff changeset	611 c = '\n'; // Convert '\r' and '\r\n' to '\n'
cf3047cf3cd2 - Added code for parsing back quote and raw strings. aziz parents: 32 diff changeset	612 case '\n':
cf3047cf3cd2 - Added code for parsing back quote and raw strings. aziz parents: 32 diff changeset	613 ++loc;
cf3047cf3cd2 - Added code for parsing back quote and raw strings. aziz parents: 32 diff changeset	614 continue;
cf3047cf3cd2 - Added code for parsing back quote and raw strings. aziz parents: 32 diff changeset	615 case '`':
cf3047cf3cd2 - Added code for parsing back quote and raw strings. aziz parents: 32 diff changeset	616 case '"':
cf3047cf3cd2 - Added code for parsing back quote and raw strings. aziz parents: 32 diff changeset	617 if (c == delim)
cf3047cf3cd2 - Added code for parsing back quote and raw strings. aziz parents: 32 diff changeset	618 {
cf3047cf3cd2 - Added code for parsing back quote and raw strings. aziz parents: 32 diff changeset	619 ++p;
cf3047cf3cd2 - Added code for parsing back quote and raw strings. aziz parents: 32 diff changeset	620 t.pf = scanPostfix();
cf3047cf3cd2 - Added code for parsing back quote and raw strings. aziz parents: 32 diff changeset	621 Lreturn:
cf3047cf3cd2 - Added code for parsing back quote and raw strings. aziz parents: 32 diff changeset	622 t.str = buffer ~ '\0';
cf3047cf3cd2 - Added code for parsing back quote and raw strings. aziz parents: 32 diff changeset	623 t.end = p;
cf3047cf3cd2 - Added code for parsing back quote and raw strings. aziz parents: 32 diff changeset	624 return;
cf3047cf3cd2 - Added code for parsing back quote and raw strings. aziz parents: 32 diff changeset	625 }
cf3047cf3cd2 - Added code for parsing back quote and raw strings. aziz parents: 32 diff changeset	626 break;
cf3047cf3cd2 - Added code for parsing back quote and raw strings. aziz parents: 32 diff changeset	627 case LS[0]:
cf3047cf3cd2 - Added code for parsing back quote and raw strings. aziz parents: 32 diff changeset	628 if (p[1] == LS[1] && (p[2] == LS[2] \|\| p[2] == PS[2]))
cf3047cf3cd2 - Added code for parsing back quote and raw strings. aziz parents: 32 diff changeset	629 {
cf3047cf3cd2 - Added code for parsing back quote and raw strings. aziz parents: 32 diff changeset	630 // TODO: convert LS or PS to \n?
cf3047cf3cd2 - Added code for parsing back quote and raw strings. aziz parents: 32 diff changeset	631 buffer ~= p[0..3];
cf3047cf3cd2 - Added code for parsing back quote and raw strings. aziz parents: 32 diff changeset	632 p += 2;
cf3047cf3cd2 - Added code for parsing back quote and raw strings. aziz parents: 32 diff changeset	633 ++loc;
cf3047cf3cd2 - Added code for parsing back quote and raw strings. aziz parents: 32 diff changeset	634 continue;
cf3047cf3cd2 - Added code for parsing back quote and raw strings. aziz parents: 32 diff changeset	635 }
cf3047cf3cd2 - Added code for parsing back quote and raw strings. aziz parents: 32 diff changeset	636 break;
cf3047cf3cd2 - Added code for parsing back quote and raw strings. aziz parents: 32 diff changeset	637 case 0, _Z_:
cf3047cf3cd2 - Added code for parsing back quote and raw strings. aziz parents: 32 diff changeset	638 if (delim == 'r')
cf3047cf3cd2 - Added code for parsing back quote and raw strings. aziz parents: 32 diff changeset	639 error(MID.UnterminatedRawString);
cf3047cf3cd2 - Added code for parsing back quote and raw strings. aziz parents: 32 diff changeset	640 else
cf3047cf3cd2 - Added code for parsing back quote and raw strings. aziz parents: 32 diff changeset	641 error(MID.UnterminatedBackQuoteString);
cf3047cf3cd2 - Added code for parsing back quote and raw strings. aziz parents: 32 diff changeset	642 goto Lreturn;
cf3047cf3cd2 - Added code for parsing back quote and raw strings. aziz parents: 32 diff changeset	643 default:
cf3047cf3cd2 - Added code for parsing back quote and raw strings. aziz parents: 32 diff changeset	644 }
cf3047cf3cd2 - Added code for parsing back quote and raw strings. aziz parents: 32 diff changeset	645 buffer ~= c; // copy character to buffer
cf3047cf3cd2 - Added code for parsing back quote and raw strings. aziz parents: 32 diff changeset	646 }
cf3047cf3cd2 - Added code for parsing back quote and raw strings. aziz parents: 32 diff changeset	647 assert(0);
31 94f09f4e988e - Added struct for strings to Token with 'pf' = postfix. aziz parents: 30 diff changeset	648 }
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix. aziz parents: 30 diff changeset	649
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix. aziz parents: 30 diff changeset	650 void scanHexStringLiteral(ref Token t)
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix. aziz parents: 30 diff changeset	651 {
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix. aziz parents: 30 diff changeset	652 assert(p[0] == 'x' && p[1] == '"');
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix. aziz parents: 30 diff changeset	653 p+=2;
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix. aziz parents: 30 diff changeset	654 t.type = TOK.String;
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix. aziz parents: 30 diff changeset	655
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix. aziz parents: 30 diff changeset	656 uint c;
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix. aziz parents: 30 diff changeset	657 ubyte[] buffer;
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix. aziz parents: 30 diff changeset	658 ubyte h; // hex number
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix. aziz parents: 30 diff changeset	659 uint n; // number of hex digits
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix. aziz parents: 30 diff changeset	660 MID mid;
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix. aziz parents: 30 diff changeset	661
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix. aziz parents: 30 diff changeset	662 while (1)
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix. aziz parents: 30 diff changeset	663 {
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix. aziz parents: 30 diff changeset	664 c = *p++;
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix. aziz parents: 30 diff changeset	665 switch (c)
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix. aziz parents: 30 diff changeset	666 {
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix. aziz parents: 30 diff changeset	667 case '"':
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix. aziz parents: 30 diff changeset	668 if (n & 1)
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix. aziz parents: 30 diff changeset	669 {
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix. aziz parents: 30 diff changeset	670 mid = MID.OddNumberOfDigitsInHexString;
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix. aziz parents: 30 diff changeset	671 error(mid);
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix. aziz parents: 30 diff changeset	672 }
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix. aziz parents: 30 diff changeset	673 t.str = cast(string) buffer;
33 cf3047cf3cd2 - Added code for parsing back quote and raw strings. aziz parents: 32 diff changeset	674 t.pf = scanPostfix();
31 94f09f4e988e - Added struct for strings to Token with 'pf' = postfix. aziz parents: 30 diff changeset	675 t.end = p;
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix. aziz parents: 30 diff changeset	676 return;
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix. aziz parents: 30 diff changeset	677 case '\r':
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix. aziz parents: 30 diff changeset	678 if (*p == '\n')
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix. aziz parents: 30 diff changeset	679 ++p;
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix. aziz parents: 30 diff changeset	680 case '\n':
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix. aziz parents: 30 diff changeset	681 ++loc;
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix. aziz parents: 30 diff changeset	682 continue;
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix. aziz parents: 30 diff changeset	683 case LS[0]:
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix. aziz parents: 30 diff changeset	684 if (*p == LS[1] && (p[1] == LS[2] \|\| p[1] == PS[2])) {
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix. aziz parents: 30 diff changeset	685 p += 2;
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix. aziz parents: 30 diff changeset	686 ++loc;
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix. aziz parents: 30 diff changeset	687 }
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix. aziz parents: 30 diff changeset	688 continue;
32 d7011daa4740 - Added missing commas after the items in the messages table. aziz parents: 31 diff changeset	689 case 0, _Z_:
31 94f09f4e988e - Added struct for strings to Token with 'pf' = postfix. aziz parents: 30 diff changeset	690 mid = MID.UnterminatedHexString;
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix. aziz parents: 30 diff changeset	691 goto Lerr;
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix. aziz parents: 30 diff changeset	692 default:
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix. aziz parents: 30 diff changeset	693 if (ishexad(c))
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix. aziz parents: 30 diff changeset	694 {
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix. aziz parents: 30 diff changeset	695 if (c <= '9')
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix. aziz parents: 30 diff changeset	696 c -= '0';
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix. aziz parents: 30 diff changeset	697 else if (c <= 'F')
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix. aziz parents: 30 diff changeset	698 c -= 'A' - 10;
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix. aziz parents: 30 diff changeset	699 else
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix. aziz parents: 30 diff changeset	700 c -= 'a' - 10;
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix. aziz parents: 30 diff changeset	701
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix. aziz parents: 30 diff changeset	702 if (n & 1)
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix. aziz parents: 30 diff changeset	703 {
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix. aziz parents: 30 diff changeset	704 h <<= 4;
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix. aziz parents: 30 diff changeset	705 h \|= c;
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix. aziz parents: 30 diff changeset	706 buffer ~= h;
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix. aziz parents: 30 diff changeset	707 }
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix. aziz parents: 30 diff changeset	708 else
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix. aziz parents: 30 diff changeset	709 h = c;
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix. aziz parents: 30 diff changeset	710 ++n;
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix. aziz parents: 30 diff changeset	711 continue;
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix. aziz parents: 30 diff changeset	712 }
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix. aziz parents: 30 diff changeset	713 else if (isspace(c))
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix. aziz parents: 30 diff changeset	714 continue;
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix. aziz parents: 30 diff changeset	715 mid = MID.NonHexCharInHexString;
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix. aziz parents: 30 diff changeset	716 goto Lerr;
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix. aziz parents: 30 diff changeset	717 }
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix. aziz parents: 30 diff changeset	718 }
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix. aziz parents: 30 diff changeset	719
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix. aziz parents: 30 diff changeset	720 return;
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix. aziz parents: 30 diff changeset	721 Lerr:
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix. aziz parents: 30 diff changeset	722 error(mid);
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix. aziz parents: 30 diff changeset	723 t.pf = 0;
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix. aziz parents: 30 diff changeset	724 t.end = p;
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix. aziz parents: 30 diff changeset	725 }
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix. aziz parents: 30 diff changeset	726
15 c70c028e47dd - Started implementation of lexing numbers. aziz parents: 14 diff changeset	727 void scanNumber(ref Token t)
c70c028e47dd - Started implementation of lexing numbers. aziz parents: 14 diff changeset	728 {
c70c028e47dd - Started implementation of lexing numbers. aziz parents: 14 diff changeset	729 while (isdigit(*++p)) {}
c70c028e47dd - Started implementation of lexing numbers. aziz parents: 14 diff changeset	730 t.type = TOK.Number;
c70c028e47dd - Started implementation of lexing numbers. aziz parents: 14 diff changeset	731 t.end = p;
30 426767b94635 - Added code for parsing the '#line' special token. aziz parents: 29 diff changeset	732 t._uint = toInt(t.span);
426767b94635 - Added code for parsing the '#line' special token. aziz parents: 29 diff changeset	733 }
426767b94635 - Added code for parsing the '#line' special token. aziz parents: 29 diff changeset	734
426767b94635 - Added code for parsing the '#line' special token. aziz parents: 29 diff changeset	735 /// Scan special token: #line Integer [Filespec] EndOfLine
426767b94635 - Added code for parsing the '#line' special token. aziz parents: 29 diff changeset	736 void scanSpecialToken()
426767b94635 - Added code for parsing the '#line' special token. aziz parents: 29 diff changeset	737 {
426767b94635 - Added code for parsing the '#line' special token. aziz parents: 29 diff changeset	738 MID mid;
426767b94635 - Added code for parsing the '#line' special token. aziz parents: 29 diff changeset	739 Token t;
426767b94635 - Added code for parsing the '#line' special token. aziz parents: 29 diff changeset	740
426767b94635 - Added code for parsing the '#line' special token. aziz parents: 29 diff changeset	741 scan(t);
426767b94635 - Added code for parsing the '#line' special token. aziz parents: 29 diff changeset	742 if (!(t.type == TOK.Identifier && t.span == "line")) {
426767b94635 - Added code for parsing the '#line' special token. aziz parents: 29 diff changeset	743 mid = MID.ExpectedIdentifierLine;
426767b94635 - Added code for parsing the '#line' special token. aziz parents: 29 diff changeset	744 goto Lerr;
426767b94635 - Added code for parsing the '#line' special token. aziz parents: 29 diff changeset	745 }
426767b94635 - Added code for parsing the '#line' special token. aziz parents: 29 diff changeset	746
426767b94635 - Added code for parsing the '#line' special token. aziz parents: 29 diff changeset	747 scan(t);
426767b94635 - Added code for parsing the '#line' special token. aziz parents: 29 diff changeset	748 if (t.type == TOK.Number)
426767b94635 - Added code for parsing the '#line' special token. aziz parents: 29 diff changeset	749 loc = t._uint - 1;
426767b94635 - Added code for parsing the '#line' special token. aziz parents: 29 diff changeset	750
426767b94635 - Added code for parsing the '#line' special token. aziz parents: 29 diff changeset	751 uint loc = this.loc;
426767b94635 - Added code for parsing the '#line' special token. aziz parents: 29 diff changeset	752
426767b94635 - Added code for parsing the '#line' special token. aziz parents: 29 diff changeset	753 char* wsstart = t.end;
426767b94635 - Added code for parsing the '#line' special token. aziz parents: 29 diff changeset	754
426767b94635 - Added code for parsing the '#line' special token. aziz parents: 29 diff changeset	755 bool hasNewline(char* end)
426767b94635 - Added code for parsing the '#line' special token. aziz parents: 29 diff changeset	756 {
426767b94635 - Added code for parsing the '#line' special token. aziz parents: 29 diff changeset	757 alias wsstart p;
426767b94635 - Added code for parsing the '#line' special token. aziz parents: 29 diff changeset	758 uint c;
426767b94635 - Added code for parsing the '#line' special token. aziz parents: 29 diff changeset	759 for(; p != end; c = *++p)
426767b94635 - Added code for parsing the '#line' special token. aziz parents: 29 diff changeset	760 if (c == '\n' \|\| c == '\r' \|\| c == LS[0] && p[1] == LS[1] && (p[2] == LS[2] \|\| p[2] == PS[2])) {
426767b94635 - Added code for parsing the '#line' special token. aziz parents: 29 diff changeset	761 mid = MID.NewlineInSpecialToken;
426767b94635 - Added code for parsing the '#line' special token. aziz parents: 29 diff changeset	762 return true;
426767b94635 - Added code for parsing the '#line' special token. aziz parents: 29 diff changeset	763 }
426767b94635 - Added code for parsing the '#line' special token. aziz parents: 29 diff changeset	764 return false;
426767b94635 - Added code for parsing the '#line' special token. aziz parents: 29 diff changeset	765 }
426767b94635 - Added code for parsing the '#line' special token. aziz parents: 29 diff changeset	766
426767b94635 - Added code for parsing the '#line' special token. aziz parents: 29 diff changeset	767 peek(t);
426767b94635 - Added code for parsing the '#line' special token. aziz parents: 29 diff changeset	768
426767b94635 - Added code for parsing the '#line' special token. aziz parents: 29 diff changeset	769 if (t.type == TOK.String)
426767b94635 - Added code for parsing the '#line' special token. aziz parents: 29 diff changeset	770 {
426767b94635 - Added code for parsing the '#line' special token. aziz parents: 29 diff changeset	771 // Check whole token with preceding whitespace for newline.
426767b94635 - Added code for parsing the '#line' special token. aziz parents: 29 diff changeset	772 if (hasNewline(t.end))
426767b94635 - Added code for parsing the '#line' special token. aziz parents: 29 diff changeset	773 goto Lerr;
426767b94635 - Added code for parsing the '#line' special token. aziz parents: 29 diff changeset	774 fileName = t.span[1..$-1]; // contents of "..."
426767b94635 - Added code for parsing the '#line' special token. aziz parents: 29 diff changeset	775 p = t.end;
426767b94635 - Added code for parsing the '#line' special token. aziz parents: 29 diff changeset	776 }
426767b94635 - Added code for parsing the '#line' special token. aziz parents: 29 diff changeset	777 else if (t.type == TOK.Identifier && t.span == "__FILE__")
426767b94635 - Added code for parsing the '#line' special token. aziz parents: 29 diff changeset	778 {
426767b94635 - Added code for parsing the '#line' special token. aziz parents: 29 diff changeset	779 // Check preceding whitespace for newline.
426767b94635 - Added code for parsing the '#line' special token. aziz parents: 29 diff changeset	780 if (hasNewline(t.start))
426767b94635 - Added code for parsing the '#line' special token. aziz parents: 29 diff changeset	781 goto Lerr;
426767b94635 - Added code for parsing the '#line' special token. aziz parents: 29 diff changeset	782 p = t.end;
426767b94635 - Added code for parsing the '#line' special token. aziz parents: 29 diff changeset	783 }
426767b94635 - Added code for parsing the '#line' special token. aziz parents: 29 diff changeset	784
426767b94635 - Added code for parsing the '#line' special token. aziz parents: 29 diff changeset	785 uint c;
426767b94635 - Added code for parsing the '#line' special token. aziz parents: 29 diff changeset	786 while (1)
426767b94635 - Added code for parsing the '#line' special token. aziz parents: 29 diff changeset	787 {
426767b94635 - Added code for parsing the '#line' special token. aziz parents: 29 diff changeset	788 c = *p++;
426767b94635 - Added code for parsing the '#line' special token. aziz parents: 29 diff changeset	789 if (isspace(c))
426767b94635 - Added code for parsing the '#line' special token. aziz parents: 29 diff changeset	790 continue;
426767b94635 - Added code for parsing the '#line' special token. aziz parents: 29 diff changeset	791
426767b94635 - Added code for parsing the '#line' special token. aziz parents: 29 diff changeset	792 if (c == '\n' \|\| c == '\r' \|\| c == 0 \|\|
426767b94635 - Added code for parsing the '#line' special token. aziz parents: 29 diff changeset	793 c == LS[0] && p[1] == LS[1] && (p[2] == LS[2] \|\| p[2] == PS[2]))
426767b94635 - Added code for parsing the '#line' special token. aziz parents: 29 diff changeset	794 break;
426767b94635 - Added code for parsing the '#line' special token. aziz parents: 29 diff changeset	795 else {
426767b94635 - Added code for parsing the '#line' special token. aziz parents: 29 diff changeset	796 mid = MID.UnterminatedSpecialToken;
426767b94635 - Added code for parsing the '#line' special token. aziz parents: 29 diff changeset	797 goto Lerr;
426767b94635 - Added code for parsing the '#line' special token. aziz parents: 29 diff changeset	798 }
426767b94635 - Added code for parsing the '#line' special token. aziz parents: 29 diff changeset	799 }
426767b94635 - Added code for parsing the '#line' special token. aziz parents: 29 diff changeset	800
426767b94635 - Added code for parsing the '#line' special token. aziz parents: 29 diff changeset	801 this.loc = loc;
426767b94635 - Added code for parsing the '#line' special token. aziz parents: 29 diff changeset	802 return;
426767b94635 - Added code for parsing the '#line' special token. aziz parents: 29 diff changeset	803 Lerr:
426767b94635 - Added code for parsing the '#line' special token. aziz parents: 29 diff changeset	804 error(mid);
15 c70c028e47dd - Started implementation of lexing numbers. aziz parents: 14 diff changeset	805 }
c70c028e47dd - Started implementation of lexing numbers. aziz parents: 14 diff changeset	806
12 0989206cf73c - Added code to decode Unicode characters in identifiers. aziz parents: 11 diff changeset	807 uint decodeUTF()
0989206cf73c - Added code to decode Unicode characters in identifiers. aziz parents: 11 diff changeset	808 {
0989206cf73c - Added code to decode Unicode characters in identifiers. aziz parents: 11 diff changeset	809 assert(*p & 128);
0989206cf73c - Added code to decode Unicode characters in identifiers. aziz parents: 11 diff changeset	810 size_t idx;
0989206cf73c - Added code to decode Unicode characters in identifiers. aziz parents: 11 diff changeset	811 uint d;
0989206cf73c - Added code to decode Unicode characters in identifiers. aziz parents: 11 diff changeset	812 d = std.utf.decode(p[0 .. end-p], idx);
0989206cf73c - Added code to decode Unicode characters in identifiers. aziz parents: 11 diff changeset	813 p += idx -1;
0989206cf73c - Added code to decode Unicode characters in identifiers. aziz parents: 11 diff changeset	814 return d;
0989206cf73c - Added code to decode Unicode characters in identifiers. aziz parents: 11 diff changeset	815 }
0989206cf73c - Added code to decode Unicode characters in identifiers. aziz parents: 11 diff changeset	816
28 3a9daccf7d96 - Added table for identifiers to Lexer. aziz parents: 27 diff changeset	817 void loadKeywords()
3a9daccf7d96 - Added table for identifiers to Lexer. aziz parents: 27 diff changeset	818 {
3a9daccf7d96 - Added table for identifiers to Lexer. aziz parents: 27 diff changeset	819 foreach(k; keywords)
3a9daccf7d96 - Added table for identifiers to Lexer. aziz parents: 27 diff changeset	820 idtable[k.str] = k;
3a9daccf7d96 - Added table for identifiers to Lexer. aziz parents: 27 diff changeset	821 }
3a9daccf7d96 - Added table for identifiers to Lexer. aziz parents: 27 diff changeset	822
18 c48d2125f1e2 - Moved code for scanning character literals to separate function. aziz parents: 17 diff changeset	823 void error(MID id)
c48d2125f1e2 - Moved code for scanning character literals to separate function. aziz parents: 17 diff changeset	824 {
c48d2125f1e2 - Moved code for scanning character literals to separate function. aziz parents: 17 diff changeset	825 errors ~= new Problem(Problem.Type.Lexer, id, loc);
c48d2125f1e2 - Moved code for scanning character literals to separate function. aziz parents: 17 diff changeset	826 }
c48d2125f1e2 - Moved code for scanning character literals to separate function. aziz parents: 17 diff changeset	827
3 4bbce78bfb1e - Added TOK enum. aziz parents: 2 diff changeset	828 public TOK nextToken()
4bbce78bfb1e - Added TOK enum. aziz parents: 2 diff changeset	829 {
4 92df59b1ec4a - Started implementation of scan(). aziz parents: 3 diff changeset	830 scan(this.token);
92df59b1ec4a - Started implementation of scan(). aziz parents: 3 diff changeset	831 return this.token.type;
92df59b1ec4a - Started implementation of scan(). aziz parents: 3 diff changeset	832 }
92df59b1ec4a - Started implementation of scan(). aziz parents: 3 diff changeset	833
92df59b1ec4a - Started implementation of scan(). aziz parents: 3 diff changeset	834 Token[] getTokens()
92df59b1ec4a - Started implementation of scan(). aziz parents: 3 diff changeset	835 {
92df59b1ec4a - Started implementation of scan(). aziz parents: 3 diff changeset	836 Token[] tokens;
92df59b1ec4a - Started implementation of scan(). aziz parents: 3 diff changeset	837 while (nextToken() != TOK.EOF)
92df59b1ec4a - Started implementation of scan(). aziz parents: 3 diff changeset	838 tokens ~= this.token;
92df59b1ec4a - Started implementation of scan(). aziz parents: 3 diff changeset	839 tokens ~= this.token;
92df59b1ec4a - Started implementation of scan(). aziz parents: 3 diff changeset	840 return tokens;
3 4bbce78bfb1e - Added TOK enum. aziz parents: 2 diff changeset	841 }
5 79b4e8848794 - Started writing XML generator. aziz parents: 4 diff changeset	842 }
33 cf3047cf3cd2 - Added code for parsing back quote and raw strings. aziz parents: 32 diff changeset	843 `fdfdfg

Mercurial > projects > dil

annotate trunk/src/Lexer.d @ 33:cf3047cf3cd2