Mercurial > projects > dil

annotate trunk/src/dil/Lexer.d @ 432:3ead178e0662

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Added message MID.InvalidUnicodeEscapeSequence.
author Aziz K?ksal <aziz.koeksal@gmail.com>
date Thu, 04 Oct 2007 16:50:59 +0200
parents e6c759e151cd
children 063cd016c913
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
0
8ba2570de175 Initial import.
aziz
parents:
diff changeset
1 /++
8ba2570de175 Initial import.
aziz
parents:
diff changeset
2 Author: Aziz Köksal
249
32d354584b28 - Upgraded license notices to GPL3.
aziz
parents: 239
diff changeset
3 License: GPL3
0
8ba2570de175 Initial import.
aziz
parents:
diff changeset
4 +/
326
4a7359b88c11 - Added package 'dil' to module declarations.
aziz
parents: 325
diff changeset
5 module dil.Lexer;
327
a48a987f7515 - Added package dil to import declarations.
aziz
parents: 326
diff changeset
6 import dil.Token;
a48a987f7515 - Added package dil to import declarations.
aziz
parents: 326
diff changeset
7 import dil.Information;
a48a987f7515 - Added package dil to import declarations.
aziz
parents: 326
diff changeset
8 import dil.Keywords;
a48a987f7515 - Added package dil to import declarations.
aziz
parents: 326
diff changeset
9 import dil.Identifier;
a48a987f7515 - Added package dil to import declarations.
aziz
parents: 326
diff changeset
10 import dil.Messages;
a48a987f7515 - Added package dil to import declarations.
aziz
parents: 326
diff changeset
11 import dil.HtmlEntities;
343
95f1b6e43214 - Removed TOK.Special and added an own entry for each special token.
aziz
parents: 327
diff changeset
12 import dil.Settings;
391
33b566df6af4 Migrated project to Tango.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 390
diff changeset
13 import tango.stdc.stdlib : strtof, strtod, strtold;
33b566df6af4 Migrated project to Tango.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 390
diff changeset
14 import tango.stdc.errno : errno, ERANGE;
33b566df6af4 Migrated project to Tango.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 390
diff changeset
15 import tango.stdc.time : time_t, time, ctime;
33b566df6af4 Migrated project to Tango.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 390
diff changeset
16 import tango.stdc.string : strlen;
4
92df59b1ec4a - Started implementation of scan().
aziz
parents: 3
diff changeset
17 import std.utf;
92df59b1ec4a - Started implementation of scan().
aziz
parents: 3
diff changeset
18 import std.uni;
391
33b566df6af4 Migrated project to Tango.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 390
diff changeset
19 import common;
0
8ba2570de175 Initial import.
aziz
parents:
diff changeset
20
430
e6c759e151cd Fixed a few things regarding encoding/decoding UTF-8 sequences.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 426
diff changeset
21 const char[3] LS = \u2028; /// Line separator.
e6c759e151cd Fixed a few things regarding encoding/decoding UTF-8 sequences.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 426
diff changeset
22 const char[3] PS = \u2029; /// Paragraph separator.
11
dffcdaa7c47a - Added Unicode line and paragraph separators.
aziz
parents: 10
diff changeset
23
18
c48d2125f1e2 - Moved code for scanning character literals to separate function.
aziz
parents: 17
diff changeset
24 const dchar LSd = 0x2028;
c48d2125f1e2 - Moved code for scanning character literals to separate function.
aziz
parents: 17
diff changeset
25 const dchar PSd = 0x2029;
c48d2125f1e2 - Moved code for scanning character literals to separate function.
aziz
parents: 17
diff changeset
26
430
e6c759e151cd Fixed a few things regarding encoding/decoding UTF-8 sequences.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 426
diff changeset
27 /// U+FFFD = �. Used to replace invalid Unicode characters.
e6c759e151cd Fixed a few things regarding encoding/decoding UTF-8 sequences.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 426
diff changeset
28 const dchar REPLACEMENT_CHAR = '\uFFFD';
e6c759e151cd Fixed a few things regarding encoding/decoding UTF-8 sequences.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 426
diff changeset
29
32
d7011daa4740 - Added missing commas after the items in the messages table.
aziz
parents: 31
diff changeset
30 const uint _Z_ = 26; /// Control+Z
d7011daa4740 - Added missing commas after the items in the messages table.
aziz
parents: 31
diff changeset
31
0
8ba2570de175 Initial import.
aziz
parents:
diff changeset
32 class Lexer
8ba2570de175 Initial import.
aziz
parents:
diff changeset
33 {
239
7911f6a92e6e - Added 'new' and 'delete' declarations to Token and uncommented next and prev members. Added HEAD to TOK.
aziz
parents: 223
diff changeset
34 Token* head; /// The head of the doubly linked token list.
344
757c86e2c3cc - Added member tail and destructor method to Lexer.
aziz
parents: 343
diff changeset
35 Token* tail; /// The tail of the linked list. Set in scan().
239
7911f6a92e6e - Added 'new' and 'delete' declarations to Token and uncommented next and prev members. Added HEAD to TOK.
aziz
parents: 223
diff changeset
36 Token* token; /// Points to the current token in the token list.
422
ad7977fe315a Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 420
diff changeset
37 string text; /// The source text.
ad7977fe315a Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 420
diff changeset
38 char[] filePath; /// Path to the source file.
239
7911f6a92e6e - Added 'new' and 'delete' declarations to Token and uncommented next and prev members. Added HEAD to TOK.
aziz
parents: 223
diff changeset
39 char* p; /// Points to the current character in the source text.
7911f6a92e6e - Added 'new' and 'delete' declarations to Token and uncommented next and prev members. Added HEAD to TOK.
aziz
parents: 223
diff changeset
40 char* end; /// Points one character past the end of the source text.
4
92df59b1ec4a - Started implementation of scan().
aziz
parents: 3
diff changeset
41
422
ad7977fe315a Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 420
diff changeset
42 // Members used to generate error messages:
ad7977fe315a Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 420
diff changeset
43 Information[] errors;
ad7977fe315a Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 420
diff changeset
44 char* lineBegin; /// Always points to the beginning of the current line.
371
01887f05d4b0 - Added members loc_old and loc_hline to class Lexer.
aziz
parents: 370
diff changeset
45 uint loc = 1; /// Actual line of code.
413
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
46 uint loc_old; /// Store actual line number when #line token is scanned.
371
01887f05d4b0 - Added members loc_old and loc_hline to class Lexer.
aziz
parents: 370
diff changeset
47 uint loc_hline; /// Line number set by #line.
422
ad7977fe315a Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 420
diff changeset
48 uint inTokenString; /// > 0 if inside q{ }
ad7977fe315a Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 420
diff changeset
49 Location errorLoc;
103
511a1aa25896 - Added reportErrors member to Lexer. Moved peek() down and rewrote it a bit making use of reportErrors. error() uses reportErrors too.
aziz
parents: 94
diff changeset
50
28
3a9daccf7d96 - Added table for identifiers to Lexer.
aziz
parents: 27
diff changeset
51 Identifier[string] idtable;
3a9daccf7d96 - Added table for identifiers to Lexer.
aziz
parents: 27
diff changeset
52
412
fb31af0fda73 Added struct Location, and token2LocTable to Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 411
diff changeset
53 version(token2LocTable)
fb31af0fda73 Added struct Location, and token2LocTable to Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 411
diff changeset
54 /// Maps every token that starts a new line to a Location.
fb31af0fda73 Added struct Location, and token2LocTable to Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 411
diff changeset
55 Location[Token*] token2LocTable;
fb31af0fda73 Added struct Location, and token2LocTable to Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 411
diff changeset
56
422
ad7977fe315a Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 420
diff changeset
57 this(string text, string filePath)
4
92df59b1ec4a - Started implementation of scan().
aziz
parents: 3
diff changeset
58 {
422
ad7977fe315a Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 420
diff changeset
59 this.filePath = filePath;
30
426767b94635 - Added code for parsing the '#line' special token.
aziz
parents: 29
diff changeset
60
4
92df59b1ec4a - Started implementation of scan().
aziz
parents: 3
diff changeset
61 this.text = text;
397
c99f8aeb7b4a Empty source files are handled correctly now.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 396
diff changeset
62 if (text.length == 0 || text[$-1] != 0)
39
69b940398d7b - Added unittest to test correct parsing of operator tokens.
aziz
parents: 38
diff changeset
63 {
69b940398d7b - Added unittest to test correct parsing of operator tokens.
aziz
parents: 38
diff changeset
64 this.text.length = this.text.length + 1;
69b940398d7b - Added unittest to test correct parsing of operator tokens.
aziz
parents: 38
diff changeset
65 this.text[$-1] = 0;
69b940398d7b - Added unittest to test correct parsing of operator tokens.
aziz
parents: 38
diff changeset
66 }
4
92df59b1ec4a - Started implementation of scan().
aziz
parents: 3
diff changeset
67
92df59b1ec4a - Started implementation of scan().
aziz
parents: 3
diff changeset
68 this.p = this.text.ptr;
92df59b1ec4a - Started implementation of scan().
aziz
parents: 3
diff changeset
69 this.end = this.p + this.text.length;
422
ad7977fe315a Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 420
diff changeset
70 this.lineBegin = this.p;
ad7977fe315a Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 420
diff changeset
71 this.errorLoc = new Location(filePath, 1, this.lineBegin, this.lineBegin);
ad7977fe315a Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 420
diff changeset
72 loadKeywords(this.idtable);
239
7911f6a92e6e - Added 'new' and 'delete' declarations to Token and uncommented next and prev members. Added HEAD to TOK.
aziz
parents: 223
diff changeset
73
7911f6a92e6e - Added 'new' and 'delete' declarations to Token and uncommented next and prev members. Added HEAD to TOK.
aziz
parents: 223
diff changeset
74 this.head = new Token;
7911f6a92e6e - Added 'new' and 'delete' declarations to Token and uncommented next and prev members. Added HEAD to TOK.
aziz
parents: 223
diff changeset
75 this.head.type = TOK.HEAD;
7911f6a92e6e - Added 'new' and 'delete' declarations to Token and uncommented next and prev members. Added HEAD to TOK.
aziz
parents: 223
diff changeset
76 this.token = this.head;
315
29c33ce6c5bb - Added method scanShebang to class Lexer.
aziz
parents: 309
diff changeset
77 scanShebang();
412
fb31af0fda73 Added struct Location, and token2LocTable to Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 411
diff changeset
78 version(token2LocTable)
fb31af0fda73 Added struct Location, and token2LocTable to Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 411
diff changeset
79 {
fb31af0fda73 Added struct Location, and token2LocTable to Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 411
diff changeset
80 // Add first token to table.
fb31af0fda73 Added struct Location, and token2LocTable to Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 411
diff changeset
81 auto firstToken = this.head;
fb31af0fda73 Added struct Location, and token2LocTable to Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 411
diff changeset
82 peek(firstToken);
420
ce644d724d87 Made Location a class and extended it with new members.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 419
diff changeset
83 token2LocTable[firstToken] = new Location(1, null);
412
fb31af0fda73 Added struct Location, and token2LocTable to Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 411
diff changeset
84 }
315
29c33ce6c5bb - Added method scanShebang to class Lexer.
aziz
parents: 309
diff changeset
85 }
29c33ce6c5bb - Added method scanShebang to class Lexer.
aziz
parents: 309
diff changeset
86
344
757c86e2c3cc - Added member tail and destructor method to Lexer.
aziz
parents: 343
diff changeset
87 ~this()
757c86e2c3cc - Added member tail and destructor method to Lexer.
aziz
parents: 343
diff changeset
88 {
757c86e2c3cc - Added member tail and destructor method to Lexer.
aziz
parents: 343
diff changeset
89 auto token = head.next;
418
7354f15cd5e9 Applied some fixes to the Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 416
diff changeset
90 while (token !is null)
344
757c86e2c3cc - Added member tail and destructor method to Lexer.
aziz
parents: 343
diff changeset
91 {
757c86e2c3cc - Added member tail and destructor method to Lexer.
aziz
parents: 343
diff changeset
92 assert(token.type == TOK.EOF ? token == tail && token.next is null : 1);
757c86e2c3cc - Added member tail and destructor method to Lexer.
aziz
parents: 343
diff changeset
93 delete token.prev;
757c86e2c3cc - Added member tail and destructor method to Lexer.
aziz
parents: 343
diff changeset
94 token = token.next;
418
7354f15cd5e9 Applied some fixes to the Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 416
diff changeset
95 }
344
757c86e2c3cc - Added member tail and destructor method to Lexer.
aziz
parents: 343
diff changeset
96 delete tail;
757c86e2c3cc - Added member tail and destructor method to Lexer.
aziz
parents: 343
diff changeset
97 }
757c86e2c3cc - Added member tail and destructor method to Lexer.
aziz
parents: 343
diff changeset
98
315
29c33ce6c5bb - Added method scanShebang to class Lexer.
aziz
parents: 309
diff changeset
99 void scanShebang()
29c33ce6c5bb - Added method scanShebang to class Lexer.
aziz
parents: 309
diff changeset
100 {
29c33ce6c5bb - Added method scanShebang to class Lexer.
aziz
parents: 309
diff changeset
101 if (*p == '#' && p[1] == '!')
29c33ce6c5bb - Added method scanShebang to class Lexer.
aziz
parents: 309
diff changeset
102 {
29c33ce6c5bb - Added method scanShebang to class Lexer.
aziz
parents: 309
diff changeset
103 Token* t = new Token;
29c33ce6c5bb - Added method scanShebang to class Lexer.
aziz
parents: 309
diff changeset
104 t.start = p;
29c33ce6c5bb - Added method scanShebang to class Lexer.
aziz
parents: 309
diff changeset
105 t.type = TOK.Shebang;
29c33ce6c5bb - Added method scanShebang to class Lexer.
aziz
parents: 309
diff changeset
106 ++p;
403
e7228859d865 Fixed the way newlines are handled after #! and #line tokens.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 402
diff changeset
107 assert(*p == '!');
315
29c33ce6c5bb - Added method scanShebang to class Lexer.
aziz
parents: 309
diff changeset
108 while (1)
29c33ce6c5bb - Added method scanShebang to class Lexer.
aziz
parents: 309
diff changeset
109 {
403
e7228859d865 Fixed the way newlines are handled after #! and #line tokens.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 402
diff changeset
110 t.end = ++p;
e7228859d865 Fixed the way newlines are handled after #! and #line tokens.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 402
diff changeset
111 switch (*p)
315
29c33ce6c5bb - Added method scanShebang to class Lexer.
aziz
parents: 309
diff changeset
112 {
413
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
113 case '\r', '\n', 0, _Z_:
315
29c33ce6c5bb - Added method scanShebang to class Lexer.
aziz
parents: 309
diff changeset
114 break;
418
7354f15cd5e9 Applied some fixes to the Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 416
diff changeset
115 case LS[0]:
7354f15cd5e9 Applied some fixes to the Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 416
diff changeset
116 if (p[1] == LS[1] && (p[2] == LS[2] || p[2] == PS[2]))
7354f15cd5e9 Applied some fixes to the Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 416
diff changeset
117 break;
315
29c33ce6c5bb - Added method scanShebang to class Lexer.
aziz
parents: 309
diff changeset
118 default:
383
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
119 if (*p & 128)
418
7354f15cd5e9 Applied some fixes to the Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 416
diff changeset
120 decodeUTF8();
315
29c33ce6c5bb - Added method scanShebang to class Lexer.
aziz
parents: 309
diff changeset
121 continue;
29c33ce6c5bb - Added method scanShebang to class Lexer.
aziz
parents: 309
diff changeset
122 }
29c33ce6c5bb - Added method scanShebang to class Lexer.
aziz
parents: 309
diff changeset
123 break; // Exit loop.
29c33ce6c5bb - Added method scanShebang to class Lexer.
aziz
parents: 309
diff changeset
124 }
403
e7228859d865 Fixed the way newlines are handled after #! and #line tokens.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 402
diff changeset
125 // Reset p. The newline will be scanned as whitespace in scan().
e7228859d865 Fixed the way newlines are handled after #! and #line tokens.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 402
diff changeset
126 p = t.end;
315
29c33ce6c5bb - Added method scanShebang to class Lexer.
aziz
parents: 309
diff changeset
127 this.head.next = t;
29c33ce6c5bb - Added method scanShebang to class Lexer.
aziz
parents: 309
diff changeset
128 t.prev = this.head;
29c33ce6c5bb - Added method scanShebang to class Lexer.
aziz
parents: 309
diff changeset
129 }
4
92df59b1ec4a - Started implementation of scan().
aziz
parents: 3
diff changeset
130 }
92df59b1ec4a - Started implementation of scan().
aziz
parents: 3
diff changeset
131
343
95f1b6e43214 - Removed TOK.Special and added an own entry for each special token.
aziz
parents: 327
diff changeset
132 void finalizeSpecialToken(ref Token t)
95f1b6e43214 - Removed TOK.Special and added an own entry for each special token.
aziz
parents: 327
diff changeset
133 {
95f1b6e43214 - Removed TOK.Special and added an own entry for each special token.
aziz
parents: 327
diff changeset
134 assert(t.srcText[0..2] == "__");
95f1b6e43214 - Removed TOK.Special and added an own entry for each special token.
aziz
parents: 327
diff changeset
135 switch (t.type)
95f1b6e43214 - Removed TOK.Special and added an own entry for each special token.
aziz
parents: 327
diff changeset
136 {
95f1b6e43214 - Removed TOK.Special and added an own entry for each special token.
aziz
parents: 327
diff changeset
137 case TOK.FILE:
422
ad7977fe315a Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 420
diff changeset
138 t.str = this.errorLoc.filePath;
343
95f1b6e43214 - Removed TOK.Special and added an own entry for each special token.
aziz
parents: 327
diff changeset
139 break;
95f1b6e43214 - Removed TOK.Special and added an own entry for each special token.
aziz
parents: 327
diff changeset
140 case TOK.LINE:
422
ad7977fe315a Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 420
diff changeset
141 t.uint_ = this.errorLineNum(this.loc);
343
95f1b6e43214 - Removed TOK.Special and added an own entry for each special token.
aziz
parents: 327
diff changeset
142 break;
95f1b6e43214 - Removed TOK.Special and added an own entry for each special token.
aziz
parents: 327
diff changeset
143 case TOK.DATE,
95f1b6e43214 - Removed TOK.Special and added an own entry for each special token.
aziz
parents: 327
diff changeset
144 TOK.TIME,
95f1b6e43214 - Removed TOK.Special and added an own entry for each special token.
aziz
parents: 327
diff changeset
145 TOK.TIMESTAMP:
95f1b6e43214 - Removed TOK.Special and added an own entry for each special token.
aziz
parents: 327
diff changeset
146 time_t time_val;
95f1b6e43214 - Removed TOK.Special and added an own entry for each special token.
aziz
parents: 327
diff changeset
147 time(&time_val);
95f1b6e43214 - Removed TOK.Special and added an own entry for each special token.
aziz
parents: 327
diff changeset
148 char* str = ctime(&time_val);
95f1b6e43214 - Removed TOK.Special and added an own entry for each special token.
aziz
parents: 327
diff changeset
149 char[] time_str = str[0 .. strlen(str)];
95f1b6e43214 - Removed TOK.Special and added an own entry for each special token.
aziz
parents: 327
diff changeset
150 switch (t.type)
95f1b6e43214 - Removed TOK.Special and added an own entry for each special token.
aziz
parents: 327
diff changeset
151 {
95f1b6e43214 - Removed TOK.Special and added an own entry for each special token.
aziz
parents: 327
diff changeset
152 case TOK.DATE:
346
ce9e2e77743f - Fix: include one space when slicing time_str for __DATE__.
aziz
parents: 344
diff changeset
153 time_str = time_str[4..11] ~ time_str[20..24] ~ \0; break;
343
95f1b6e43214 - Removed TOK.Special and added an own entry for each special token.
aziz
parents: 327
diff changeset
154 case TOK.TIME:
95f1b6e43214 - Removed TOK.Special and added an own entry for each special token.
aziz
parents: 327
diff changeset
155 time_str = time_str[11..19] ~ \0; break;
95f1b6e43214 - Removed TOK.Special and added an own entry for each special token.
aziz
parents: 327
diff changeset
156 case TOK.TIMESTAMP:
95f1b6e43214 - Removed TOK.Special and added an own entry for each special token.
aziz
parents: 327
diff changeset
157 time_str = time_str[0..24] ~ \0; break;
95f1b6e43214 - Removed TOK.Special and added an own entry for each special token.
aziz
parents: 327
diff changeset
158 default: assert(0);
95f1b6e43214 - Removed TOK.Special and added an own entry for each special token.
aziz
parents: 327
diff changeset
159 }
95f1b6e43214 - Removed TOK.Special and added an own entry for each special token.
aziz
parents: 327
diff changeset
160 t.str = time_str;
95f1b6e43214 - Removed TOK.Special and added an own entry for each special token.
aziz
parents: 327
diff changeset
161 break;
95f1b6e43214 - Removed TOK.Special and added an own entry for each special token.
aziz
parents: 327
diff changeset
162 case TOK.VENDOR:
95f1b6e43214 - Removed TOK.Special and added an own entry for each special token.
aziz
parents: 327
diff changeset
163 t.str = VENDOR;
95f1b6e43214 - Removed TOK.Special and added an own entry for each special token.
aziz
parents: 327
diff changeset
164 break;
95f1b6e43214 - Removed TOK.Special and added an own entry for each special token.
aziz
parents: 327
diff changeset
165 case TOK.VERSION:
95f1b6e43214 - Removed TOK.Special and added an own entry for each special token.
aziz
parents: 327
diff changeset
166 t.uint_ = VERSION_MAJOR*1000 + VERSION_MINOR;
95f1b6e43214 - Removed TOK.Special and added an own entry for each special token.
aziz
parents: 327
diff changeset
167 break;
95f1b6e43214 - Removed TOK.Special and added an own entry for each special token.
aziz
parents: 327
diff changeset
168 default:
95f1b6e43214 - Removed TOK.Special and added an own entry for each special token.
aziz
parents: 327
diff changeset
169 assert(0);
95f1b6e43214 - Removed TOK.Special and added an own entry for each special token.
aziz
parents: 327
diff changeset
170 }
95f1b6e43214 - Removed TOK.Special and added an own entry for each special token.
aziz
parents: 327
diff changeset
171 }
95f1b6e43214 - Removed TOK.Special and added an own entry for each special token.
aziz
parents: 327
diff changeset
172
422
ad7977fe315a Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 420
diff changeset
173 void setLineBegin(char* p)
ad7977fe315a Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 420
diff changeset
174 {
ad7977fe315a Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 420
diff changeset
175 // Check that we can look behind one character.
ad7977fe315a Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 420
diff changeset
176 assert((p-1) >= text.ptr && p < end);
ad7977fe315a Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 420
diff changeset
177 // Check that previous character is a newline.
ad7977fe315a Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 420
diff changeset
178 assert(p[-1] == '\n' || p[-1] == '\r' ||
ad7977fe315a Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 420
diff changeset
179 p[-1] == LS[2] || p[-1] == PS[2]);
ad7977fe315a Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 420
diff changeset
180 this.lineBegin = p;
ad7977fe315a Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 420
diff changeset
181 }
ad7977fe315a Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 420
diff changeset
182
ad7977fe315a Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 420
diff changeset
183 private void scanNext(ref Token* t)
419
89e40d43065d Added new member 'p_newl' to Lexer and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 418
diff changeset
184 {
422
ad7977fe315a Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 420
diff changeset
185 assert(t !is null);
ad7977fe315a Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 420
diff changeset
186 if (t.next)
ad7977fe315a Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 420
diff changeset
187 t = t.next;
ad7977fe315a Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 420
diff changeset
188 else if (t != this.tail)
ad7977fe315a Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 420
diff changeset
189 {
ad7977fe315a Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 420
diff changeset
190 Token* new_t = new Token;
ad7977fe315a Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 420
diff changeset
191 scan(*new_t);
ad7977fe315a Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 420
diff changeset
192 new_t.prev = t;
ad7977fe315a Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 420
diff changeset
193 t.next = new_t;
ad7977fe315a Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 420
diff changeset
194 t = new_t;
ad7977fe315a Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 420
diff changeset
195 }
ad7977fe315a Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 420
diff changeset
196 }
ad7977fe315a Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 420
diff changeset
197
ad7977fe315a Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 420
diff changeset
198 void peek(ref Token* t)
ad7977fe315a Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 420
diff changeset
199 {
ad7977fe315a Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 420
diff changeset
200 scanNext(t);
ad7977fe315a Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 420
diff changeset
201 }
ad7977fe315a Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 420
diff changeset
202
ad7977fe315a Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 420
diff changeset
203 TOK nextToken()
ad7977fe315a Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 420
diff changeset
204 {
ad7977fe315a Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 420
diff changeset
205 scanNext(this.token);
ad7977fe315a Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 420
diff changeset
206 return this.token.type;
419
89e40d43065d Added new member 'p_newl' to Lexer and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 418
diff changeset
207 }
89e40d43065d Added new member 'p_newl' to Lexer and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 418
diff changeset
208
413
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
209 public void scan_(out Token t)
207
481ed2b63a49 - Added contracts to method scan().
aziz
parents: 131
diff changeset
210 in
3
4bbce78bfb1e - Added TOK enum.
aziz
parents: 2
diff changeset
211 {
207
481ed2b63a49 - Added contracts to method scan().
aziz
parents: 131
diff changeset
212 assert(text.ptr <= p && p < end);
481ed2b63a49 - Added contracts to method scan().
aziz
parents: 131
diff changeset
213 }
481ed2b63a49 - Added contracts to method scan().
aziz
parents: 131
diff changeset
214 out
481ed2b63a49 - Added contracts to method scan().
aziz
parents: 131
diff changeset
215 {
397
c99f8aeb7b4a Empty source files are handled correctly now.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 396
diff changeset
216 assert(text.ptr <= t.start && t.start < end, Token.toString(t.type));
c99f8aeb7b4a Empty source files are handled correctly now.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 396
diff changeset
217 assert(text.ptr <= t.end && t.end <= end, Token.toString(t.type));
207
481ed2b63a49 - Added contracts to method scan().
aziz
parents: 131
diff changeset
218 }
481ed2b63a49 - Added contracts to method scan().
aziz
parents: 131
diff changeset
219 body
481ed2b63a49 - Added contracts to method scan().
aziz
parents: 131
diff changeset
220 {
390
4d36eea1bbc9 Refactored Lexer.scan().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 389
diff changeset
221 // Scan whitespace.
4d36eea1bbc9 Refactored Lexer.scan().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 389
diff changeset
222 auto pws = p;
418
7354f15cd5e9 Applied some fixes to the Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 416
diff changeset
223 auto old_loc = this.loc;
390
4d36eea1bbc9 Refactored Lexer.scan().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 389
diff changeset
224 while (1)
4d36eea1bbc9 Refactored Lexer.scan().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 389
diff changeset
225 {
4d36eea1bbc9 Refactored Lexer.scan().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 389
diff changeset
226 switch (*p)
4d36eea1bbc9 Refactored Lexer.scan().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 389
diff changeset
227 {
4d36eea1bbc9 Refactored Lexer.scan().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 389
diff changeset
228 case '\r':
4d36eea1bbc9 Refactored Lexer.scan().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 389
diff changeset
229 if (p[1] == '\n')
4d36eea1bbc9 Refactored Lexer.scan().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 389
diff changeset
230 ++p;
4d36eea1bbc9 Refactored Lexer.scan().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 389
diff changeset
231 case '\n':
419
89e40d43065d Added new member 'p_newl' to Lexer and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 418
diff changeset
232 assert(*p == '\n' || *p == '\r' || *p == LS[2] || *p == PS[2]);
390
4d36eea1bbc9 Refactored Lexer.scan().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 389
diff changeset
233 ++p;
4d36eea1bbc9 Refactored Lexer.scan().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 389
diff changeset
234 ++loc;
422
ad7977fe315a Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 420
diff changeset
235 setLineBegin(p);
390
4d36eea1bbc9 Refactored Lexer.scan().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 389
diff changeset
236 continue;
4d36eea1bbc9 Refactored Lexer.scan().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 389
diff changeset
237 case LS[0]:
4d36eea1bbc9 Refactored Lexer.scan().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 389
diff changeset
238 if (p[1] == LS[1] && (p[2] == LS[2] || p[2] == PS[2]))
4d36eea1bbc9 Refactored Lexer.scan().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 389
diff changeset
239 {
412
fb31af0fda73 Added struct Location, and token2LocTable to Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 411
diff changeset
240 ++p; ++p;
fb31af0fda73 Added struct Location, and token2LocTable to Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 411
diff changeset
241 goto case '\n';
390
4d36eea1bbc9 Refactored Lexer.scan().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 389
diff changeset
242 }
4d36eea1bbc9 Refactored Lexer.scan().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 389
diff changeset
243 // goto default;
4d36eea1bbc9 Refactored Lexer.scan().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 389
diff changeset
244 default:
4d36eea1bbc9 Refactored Lexer.scan().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 389
diff changeset
245 if (!isspace(*p))
4d36eea1bbc9 Refactored Lexer.scan().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 389
diff changeset
246 break;
4d36eea1bbc9 Refactored Lexer.scan().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 389
diff changeset
247 ++p;
4d36eea1bbc9 Refactored Lexer.scan().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 389
diff changeset
248 continue;
4d36eea1bbc9 Refactored Lexer.scan().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 389
diff changeset
249 }
4d36eea1bbc9 Refactored Lexer.scan().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 389
diff changeset
250 break; // Exit loop.
4d36eea1bbc9 Refactored Lexer.scan().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 389
diff changeset
251 }
4
92df59b1ec4a - Started implementation of scan().
aziz
parents: 3
diff changeset
252
390
4d36eea1bbc9 Refactored Lexer.scan().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 389
diff changeset
253 if (p != pws)
418
7354f15cd5e9 Applied some fixes to the Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 416
diff changeset
254 {
390
4d36eea1bbc9 Refactored Lexer.scan().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 389
diff changeset
255 t.ws = pws;
418
7354f15cd5e9 Applied some fixes to the Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 416
diff changeset
256 if (old_loc != this.loc)
7354f15cd5e9 Applied some fixes to the Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 416
diff changeset
257 version(token2LocTable)
420
ce644d724d87 Made Location a class and extended it with new members.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 419
diff changeset
258 token2LocTable[&t] = new Location(loc, null);
418
7354f15cd5e9 Applied some fixes to the Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 416
diff changeset
259 }
390
4d36eea1bbc9 Refactored Lexer.scan().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 389
diff changeset
260
4d36eea1bbc9 Refactored Lexer.scan().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 389
diff changeset
261 // Scan token.
4d36eea1bbc9 Refactored Lexer.scan().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 389
diff changeset
262 uint c = *p;
4
92df59b1ec4a - Started implementation of scan().
aziz
parents: 3
diff changeset
263 {
5
79b4e8848794 - Started writing XML generator.
aziz
parents: 4
diff changeset
264 t.start = p;
16
476e8e55c1d4 - Added Whitespace to the character properties table.
aziz
parents: 15
diff changeset
265
13
e5211758b63c - Added isidbeg() function.
aziz
parents: 12
diff changeset
266 if (isidbeg(c))
5
79b4e8848794 - Started writing XML generator.
aziz
parents: 4
diff changeset
267 {
33
cf3047cf3cd2 - Added code for parsing back quote and raw strings.
aziz
parents: 32
diff changeset
268 if (c == 'r' && p[1] == '"' && ++p)
31
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix.
aziz
parents: 30
diff changeset
269 return scanRawStringLiteral(t);
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix.
aziz
parents: 30
diff changeset
270 if (c == 'x' && p[1] == '"')
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix.
aziz
parents: 30
diff changeset
271 return scanHexStringLiteral(t);
383
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
272 version(D2)
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
273 {
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
274 if (c == 'q' && p[1] == '"')
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
275 return scanDelimitedStringLiteral(t);
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
276 if (c == 'q' && p[1] == '{')
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
277 return scanTokenStringLiteral(t);
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
278 }
12
0989206cf73c - Added code to decode Unicode characters in identifiers.
aziz
parents: 11
diff changeset
279 Lidentifier:
5
79b4e8848794 - Started writing XML generator.
aziz
parents: 4
diff changeset
280 do
79b4e8848794 - Started writing XML generator.
aziz
parents: 4
diff changeset
281 { c = *++p; }
49
7f0fa15dcffc - Renamed function.
aziz
parents: 48
diff changeset
282 while (isident(c) || c & 128 && isUniAlpha(decodeUTF8()))
28
3a9daccf7d96 - Added table for identifiers to Lexer.
aziz
parents: 27
diff changeset
283
5
79b4e8848794 - Started writing XML generator.
aziz
parents: 4
diff changeset
284 t.end = p;
28
3a9daccf7d96 - Added table for identifiers to Lexer.
aziz
parents: 27
diff changeset
285
65
6c21ae79fbb3 - Renamed function Token.span to Token.srcText.
aziz
parents: 64
diff changeset
286 string str = t.srcText;
28
3a9daccf7d96 - Added table for identifiers to Lexer.
aziz
parents: 27
diff changeset
287 Identifier* id = str in idtable;
3a9daccf7d96 - Added table for identifiers to Lexer.
aziz
parents: 27
diff changeset
288
3a9daccf7d96 - Added table for identifiers to Lexer.
aziz
parents: 27
diff changeset
289 if (!id)
3a9daccf7d96 - Added table for identifiers to Lexer.
aziz
parents: 27
diff changeset
290 {
327
a48a987f7515 - Added package dil to import declarations.
aziz
parents: 326
diff changeset
291 idtable[str] = Identifier(TOK.Identifier, str);
28
3a9daccf7d96 - Added table for identifiers to Lexer.
aziz
parents: 27
diff changeset
292 id = str in idtable;
3a9daccf7d96 - Added table for identifiers to Lexer.
aziz
parents: 27
diff changeset
293 }
3a9daccf7d96 - Added table for identifiers to Lexer.
aziz
parents: 27
diff changeset
294 assert(id);
3a9daccf7d96 - Added table for identifiers to Lexer.
aziz
parents: 27
diff changeset
295 t.type = id.type;
411
cca83c0c00fd Added __EOF__ token.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 410
diff changeset
296 if (t.type == TOK.Identifier)
cca83c0c00fd Added __EOF__ token.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 410
diff changeset
297 return;
cca83c0c00fd Added __EOF__ token.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 410
diff changeset
298 if (t.type == TOK.EOF)
cca83c0c00fd Added __EOF__ token.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 410
diff changeset
299 {
cca83c0c00fd Added __EOF__ token.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 410
diff changeset
300 t.type = TOK.EOF;
cca83c0c00fd Added __EOF__ token.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 410
diff changeset
301 t.end = p;
cca83c0c00fd Added __EOF__ token.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 410
diff changeset
302 tail = &t;
cca83c0c00fd Added __EOF__ token.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 410
diff changeset
303 assert(t.srcText == "__EOF__");
cca83c0c00fd Added __EOF__ token.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 410
diff changeset
304 }
cca83c0c00fd Added __EOF__ token.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 410
diff changeset
305 else if (t.isSpecialToken)
343
95f1b6e43214 - Removed TOK.Special and added an own entry for each special token.
aziz
parents: 327
diff changeset
306 finalizeSpecialToken(t);
5
79b4e8848794 - Started writing XML generator.
aziz
parents: 4
diff changeset
307 return;
79b4e8848794 - Started writing XML generator.
aziz
parents: 4
diff changeset
308 }
79b4e8848794 - Started writing XML generator.
aziz
parents: 4
diff changeset
309
15
c70c028e47dd - Started implementation of lexing numbers.
aziz
parents: 14
diff changeset
310 if (isdigit(c))
c70c028e47dd - Started implementation of lexing numbers.
aziz
parents: 14
diff changeset
311 return scanNumber(t);
c70c028e47dd - Started implementation of lexing numbers.
aziz
parents: 14
diff changeset
312
8
d4ba94a5a282 - Parsing /* */ comments now.
aziz
parents: 7
diff changeset
313 if (c == '/')
5
79b4e8848794 - Started writing XML generator.
aziz
parents: 4
diff changeset
314 {
8
d4ba94a5a282 - Parsing /* */ comments now.
aziz
parents: 7
diff changeset
315 c = *++p;
14
cdf788d8bdaf - Parsing /= now.
aziz
parents: 13
diff changeset
316 switch(c)
5
79b4e8848794 - Started writing XML generator.
aziz
parents: 4
diff changeset
317 {
18
c48d2125f1e2 - Moved code for scanning character literals to separate function.
aziz
parents: 17
diff changeset
318 case '=':
14
cdf788d8bdaf - Parsing /= now.
aziz
parents: 13
diff changeset
319 ++p;
32
d7011daa4740 - Added missing commas after the items in the messages table.
aziz
parents: 31
diff changeset
320 t.type = TOK.DivAssign;
14
cdf788d8bdaf - Parsing /= now.
aziz
parents: 13
diff changeset
321 t.end = p;
cdf788d8bdaf - Parsing /= now.
aziz
parents: 13
diff changeset
322 return;
18
c48d2125f1e2 - Moved code for scanning character literals to separate function.
aziz
parents: 17
diff changeset
323 case '+':
413
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
324 return scanNestedComment(t);
32
d7011daa4740 - Added missing commas after the items in the messages table.
aziz
parents: 31
diff changeset
325 case '*':
413
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
326 return scanBlockComment(t);
32
d7011daa4740 - Added missing commas after the items in the messages table.
aziz
parents: 31
diff changeset
327 case '/':
d7011daa4740 - Added missing commas after the items in the messages table.
aziz
parents: 31
diff changeset
328 while (1)
7
07e45c06a024 - Parsing nested comments correctly now.
aziz
parents: 5
diff changeset
329 {
8
d4ba94a5a282 - Parsing /* */ comments now.
aziz
parents: 7
diff changeset
330 c = *++p;
32
d7011daa4740 - Added missing commas after the items in the messages table.
aziz
parents: 31
diff changeset
331 switch (c)
d7011daa4740 - Added missing commas after the items in the messages table.
aziz
parents: 31
diff changeset
332 {
413
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
333 case '\r', '\n', 0, _Z_:
11
dffcdaa7c47a - Added Unicode line and paragraph separators.
aziz
parents: 10
diff changeset
334 break;
419
89e40d43065d Added new member 'p_newl' to Lexer and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 418
diff changeset
335 case LS[0]:
89e40d43065d Added new member 'p_newl' to Lexer and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 418
diff changeset
336 if (p[1] == LS[1] && (p[2] == LS[2] || p[2] == PS[2]))
89e40d43065d Added new member 'p_newl' to Lexer and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 418
diff changeset
337 break;
32
d7011daa4740 - Added missing commas after the items in the messages table.
aziz
parents: 31
diff changeset
338 default:
413
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
339 if (c & 128)
419
89e40d43065d Added new member 'p_newl' to Lexer and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 418
diff changeset
340 decodeUTF8();
32
d7011daa4740 - Added missing commas after the items in the messages table.
aziz
parents: 31
diff changeset
341 continue;
d7011daa4740 - Added missing commas after the items in the messages table.
aziz
parents: 31
diff changeset
342 }
413
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
343 break; // Exit loop.
32
d7011daa4740 - Added missing commas after the items in the messages table.
aziz
parents: 31
diff changeset
344 }
413
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
345 t.type = TOK.Comment;
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
346 t.end = p;
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
347 return;
32
d7011daa4740 - Added missing commas after the items in the messages table.
aziz
parents: 31
diff changeset
348 default:
d7011daa4740 - Added missing commas after the items in the messages table.
aziz
parents: 31
diff changeset
349 t.type = TOK.Div;
10
3ee65d6e39c9 - Parsing // comments now.
aziz
parents: 9
diff changeset
350 t.end = p;
3ee65d6e39c9 - Parsing // comments now.
aziz
parents: 9
diff changeset
351 return;
3ee65d6e39c9 - Parsing // comments now.
aziz
parents: 9
diff changeset
352 }
5
79b4e8848794 - Started writing XML generator.
aziz
parents: 4
diff changeset
353 }
79b4e8848794 - Started writing XML generator.
aziz
parents: 4
diff changeset
354
36
3c7210a722f7 - Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents: 35
diff changeset
355 switch (c)
20
d6adfbd7c513 - Added code for parsing braces.
aziz
parents: 19
diff changeset
356 {
48
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
357 case '\'':
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
358 return scanCharacterLiteral(t);
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
359 case '`':
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
360 return scanRawStringLiteral(t);
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
361 case '"':
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
362 return scanNormalStringLiteral(t);
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
363 case '\\':
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
364 char[] buffer;
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
365 do
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
366 {
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
367 c = scanEscapeSequence();
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
368 if (c < 128)
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
369 buffer ~= c;
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
370 else
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
371 encodeUTF8(buffer, c);
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
372 } while (*p == '\\')
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
373 buffer ~= 0;
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
374 t.type = TOK.String;
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
375 t.str = buffer;
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
376 t.end = p;
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
377 return;
38
640c45aaaaee - Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents: 37
diff changeset
378 case '>': /* > >= >> >>= >>> >>>= */
640c45aaaaee - Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents: 37
diff changeset
379 c = *++p;
640c45aaaaee - Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents: 37
diff changeset
380 switch (c)
640c45aaaaee - Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents: 37
diff changeset
381 {
640c45aaaaee - Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents: 37
diff changeset
382 case '=':
640c45aaaaee - Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents: 37
diff changeset
383 t.type = TOK.GreaterEqual;
640c45aaaaee - Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents: 37
diff changeset
384 goto Lcommon;
640c45aaaaee - Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents: 37
diff changeset
385 case '>':
640c45aaaaee - Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents: 37
diff changeset
386 if (p[1] == '>')
640c45aaaaee - Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents: 37
diff changeset
387 {
640c45aaaaee - Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents: 37
diff changeset
388 ++p;
640c45aaaaee - Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents: 37
diff changeset
389 if (p[1] == '=')
640c45aaaaee - Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents: 37
diff changeset
390 { ++p;
640c45aaaaee - Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents: 37
diff changeset
391 t.type = TOK.URShiftAssign;
640c45aaaaee - Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents: 37
diff changeset
392 }
640c45aaaaee - Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents: 37
diff changeset
393 else
640c45aaaaee - Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents: 37
diff changeset
394 t.type = TOK.URShift;
640c45aaaaee - Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents: 37
diff changeset
395 }
640c45aaaaee - Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents: 37
diff changeset
396 else if (p[1] == '=')
640c45aaaaee - Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents: 37
diff changeset
397 {
640c45aaaaee - Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents: 37
diff changeset
398 ++p;
640c45aaaaee - Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents: 37
diff changeset
399 t.type = TOK.RShiftAssign;
640c45aaaaee - Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents: 37
diff changeset
400 }
640c45aaaaee - Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents: 37
diff changeset
401 else
640c45aaaaee - Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents: 37
diff changeset
402 t.type = TOK.RShift;
640c45aaaaee - Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents: 37
diff changeset
403 goto Lcommon;
640c45aaaaee - Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents: 37
diff changeset
404 default:
640c45aaaaee - Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents: 37
diff changeset
405 t.type = TOK.Greater;
640c45aaaaee - Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents: 37
diff changeset
406 goto Lcommon2;
640c45aaaaee - Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents: 37
diff changeset
407 }
640c45aaaaee - Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents: 37
diff changeset
408 assert(0);
36
3c7210a722f7 - Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents: 35
diff changeset
409 case '<': /* < <= <> <>= << <<= */
3c7210a722f7 - Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents: 35
diff changeset
410 c = *++p;
3c7210a722f7 - Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents: 35
diff changeset
411 switch (c)
3c7210a722f7 - Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents: 35
diff changeset
412 {
3c7210a722f7 - Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents: 35
diff changeset
413 case '=':
3c7210a722f7 - Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents: 35
diff changeset
414 t.type = TOK.LessEqual;
3c7210a722f7 - Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents: 35
diff changeset
415 goto Lcommon;
3c7210a722f7 - Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents: 35
diff changeset
416 case '<':
3c7210a722f7 - Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents: 35
diff changeset
417 if (p[1] == '=') {
3c7210a722f7 - Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents: 35
diff changeset
418 ++p;
3c7210a722f7 - Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents: 35
diff changeset
419 t.type = TOK.LShiftAssign;
3c7210a722f7 - Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents: 35
diff changeset
420 }
3c7210a722f7 - Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents: 35
diff changeset
421 else
3c7210a722f7 - Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents: 35
diff changeset
422 t.type = TOK.LShift;
3c7210a722f7 - Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents: 35
diff changeset
423 goto Lcommon;
3c7210a722f7 - Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents: 35
diff changeset
424 case '>':
3c7210a722f7 - Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents: 35
diff changeset
425 if (p[1] == '=') {
3c7210a722f7 - Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents: 35
diff changeset
426 ++p;
3c7210a722f7 - Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents: 35
diff changeset
427 t.type = TOK.LorEorG;
3c7210a722f7 - Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents: 35
diff changeset
428 }
3c7210a722f7 - Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents: 35
diff changeset
429 else
3c7210a722f7 - Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents: 35
diff changeset
430 t.type = TOK.LorG;
3c7210a722f7 - Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents: 35
diff changeset
431 goto Lcommon;
3c7210a722f7 - Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents: 35
diff changeset
432 default:
38
640c45aaaaee - Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents: 37
diff changeset
433 t.type = TOK.Less;
36
3c7210a722f7 - Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents: 35
diff changeset
434 goto Lcommon2;
3c7210a722f7 - Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents: 35
diff changeset
435 }
3c7210a722f7 - Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents: 35
diff changeset
436 assert(0);
37
7f3bcb97d017 - Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents: 36
diff changeset
437 case '!': /* ! !< !> !<= !>= !<> !<>= */
35
c470b9356e35 - Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents: 34
diff changeset
438 c = *++p;
c470b9356e35 - Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents: 34
diff changeset
439 switch (c)
c470b9356e35 - Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents: 34
diff changeset
440 {
c470b9356e35 - Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents: 34
diff changeset
441 case '<':
c470b9356e35 - Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents: 34
diff changeset
442 c = *++p;
c470b9356e35 - Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents: 34
diff changeset
443 if (c == '>')
c470b9356e35 - Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents: 34
diff changeset
444 {
c470b9356e35 - Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents: 34
diff changeset
445 if (p[1] == '=') {
c470b9356e35 - Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents: 34
diff changeset
446 ++p;
c470b9356e35 - Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents: 34
diff changeset
447 t.type = TOK.Unordered;
c470b9356e35 - Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents: 34
diff changeset
448 }
c470b9356e35 - Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents: 34
diff changeset
449 else
c470b9356e35 - Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents: 34
diff changeset
450 t.type = TOK.UorE;
c470b9356e35 - Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents: 34
diff changeset
451 }
c470b9356e35 - Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents: 34
diff changeset
452 else if (c == '=')
c470b9356e35 - Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents: 34
diff changeset
453 {
c470b9356e35 - Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents: 34
diff changeset
454 t.type = TOK.UorG;
c470b9356e35 - Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents: 34
diff changeset
455 }
c470b9356e35 - Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents: 34
diff changeset
456 else {
c470b9356e35 - Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents: 34
diff changeset
457 t.type = TOK.UorGorE;
c470b9356e35 - Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents: 34
diff changeset
458 goto Lcommon2;
c470b9356e35 - Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents: 34
diff changeset
459 }
c470b9356e35 - Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents: 34
diff changeset
460 goto Lcommon;
c470b9356e35 - Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents: 34
diff changeset
461 case '>':
c470b9356e35 - Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents: 34
diff changeset
462 if (p[1] == '=')
c470b9356e35 - Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents: 34
diff changeset
463 {
c470b9356e35 - Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents: 34
diff changeset
464 ++p;
c470b9356e35 - Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents: 34
diff changeset
465 t.type = TOK.UorL;
c470b9356e35 - Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents: 34
diff changeset
466 }
c470b9356e35 - Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents: 34
diff changeset
467 else
c470b9356e35 - Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents: 34
diff changeset
468 t.type = TOK.UorLorE;
c470b9356e35 - Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents: 34
diff changeset
469 goto Lcommon;
c470b9356e35 - Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents: 34
diff changeset
470 case '=':
c470b9356e35 - Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents: 34
diff changeset
471 t.type = TOK.NotEqual;
c470b9356e35 - Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents: 34
diff changeset
472 goto Lcommon;
c470b9356e35 - Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents: 34
diff changeset
473 default:
c470b9356e35 - Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents: 34
diff changeset
474 t.type = TOK.Not;
c470b9356e35 - Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents: 34
diff changeset
475 goto Lcommon2;
c470b9356e35 - Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents: 34
diff changeset
476 }
c470b9356e35 - Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents: 34
diff changeset
477 assert(0);
63
c29229fbf2f7 - Recognizing floats that start with a dot.
aziz
parents: 62
diff changeset
478 case '.': /* . .[0-9] .. ... */
22
b05fff8e2ce4 - Added code for parsing Dot, Slice and Ellipses tokens.
aziz
parents: 21
diff changeset
479 if (p[1] == '.')
b05fff8e2ce4 - Added code for parsing Dot, Slice and Ellipses tokens.
aziz
parents: 21
diff changeset
480 {
b05fff8e2ce4 - Added code for parsing Dot, Slice and Ellipses tokens.
aziz
parents: 21
diff changeset
481 ++p;
b05fff8e2ce4 - Added code for parsing Dot, Slice and Ellipses tokens.
aziz
parents: 21
diff changeset
482 if (p[1] == '.') {
b05fff8e2ce4 - Added code for parsing Dot, Slice and Ellipses tokens.
aziz
parents: 21
diff changeset
483 ++p;
b05fff8e2ce4 - Added code for parsing Dot, Slice and Ellipses tokens.
aziz
parents: 21
diff changeset
484 t.type = TOK.Ellipses;
b05fff8e2ce4 - Added code for parsing Dot, Slice and Ellipses tokens.
aziz
parents: 21
diff changeset
485 }
b05fff8e2ce4 - Added code for parsing Dot, Slice and Ellipses tokens.
aziz
parents: 21
diff changeset
486 else
b05fff8e2ce4 - Added code for parsing Dot, Slice and Ellipses tokens.
aziz
parents: 21
diff changeset
487 t.type = TOK.Slice;
b05fff8e2ce4 - Added code for parsing Dot, Slice and Ellipses tokens.
aziz
parents: 21
diff changeset
488 }
63
c29229fbf2f7 - Recognizing floats that start with a dot.
aziz
parents: 62
diff changeset
489 else if (isdigit(p[1]))
c29229fbf2f7 - Recognizing floats that start with a dot.
aziz
parents: 62
diff changeset
490 {
c29229fbf2f7 - Recognizing floats that start with a dot.
aziz
parents: 62
diff changeset
491 return scanReal(t);
c29229fbf2f7 - Recognizing floats that start with a dot.
aziz
parents: 62
diff changeset
492 }
22
b05fff8e2ce4 - Added code for parsing Dot, Slice and Ellipses tokens.
aziz
parents: 21
diff changeset
493 else
b05fff8e2ce4 - Added code for parsing Dot, Slice and Ellipses tokens.
aziz
parents: 21
diff changeset
494 t.type = TOK.Dot;
b05fff8e2ce4 - Added code for parsing Dot, Slice and Ellipses tokens.
aziz
parents: 21
diff changeset
495 goto Lcommon;
36
3c7210a722f7 - Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents: 35
diff changeset
496 case '|': /* | || |= */
23
1a7903701a3d - Added code for parsing OrAssign, OrLogical and OrBinary tokens.
aziz
parents: 22
diff changeset
497 c = *++p;
1a7903701a3d - Added code for parsing OrAssign, OrLogical and OrBinary tokens.
aziz
parents: 22
diff changeset
498 if (c == '=')
1a7903701a3d - Added code for parsing OrAssign, OrLogical and OrBinary tokens.
aziz
parents: 22
diff changeset
499 t.type = TOK.OrAssign;
1a7903701a3d - Added code for parsing OrAssign, OrLogical and OrBinary tokens.
aziz
parents: 22
diff changeset
500 else if (c == '|')
1a7903701a3d - Added code for parsing OrAssign, OrLogical and OrBinary tokens.
aziz
parents: 22
diff changeset
501 t.type = TOK.OrLogical;
26
c3d7373db241 - Added code for parsing Assign and Equal tokens.
aziz
parents: 25
diff changeset
502 else {
23
1a7903701a3d - Added code for parsing OrAssign, OrLogical and OrBinary tokens.
aziz
parents: 22
diff changeset
503 t.type = TOK.OrBinary;
26
c3d7373db241 - Added code for parsing Assign and Equal tokens.
aziz
parents: 25
diff changeset
504 goto Lcommon2;
c3d7373db241 - Added code for parsing Assign and Equal tokens.
aziz
parents: 25
diff changeset
505 }
23
1a7903701a3d - Added code for parsing OrAssign, OrLogical and OrBinary tokens.
aziz
parents: 22
diff changeset
506 goto Lcommon;
36
3c7210a722f7 - Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents: 35
diff changeset
507 case '&': /* & && &= */
24
903f91163f23 - Added code for parsing AndAssign, AndLogical and AndBinary tokens.
aziz
parents: 23
diff changeset
508 c = *++p;
903f91163f23 - Added code for parsing AndAssign, AndLogical and AndBinary tokens.
aziz
parents: 23
diff changeset
509 if (c == '=')
903f91163f23 - Added code for parsing AndAssign, AndLogical and AndBinary tokens.
aziz
parents: 23
diff changeset
510 t.type = TOK.AndAssign;
903f91163f23 - Added code for parsing AndAssign, AndLogical and AndBinary tokens.
aziz
parents: 23
diff changeset
511 else if (c == '&')
903f91163f23 - Added code for parsing AndAssign, AndLogical and AndBinary tokens.
aziz
parents: 23
diff changeset
512 t.type = TOK.AndLogical;
26
c3d7373db241 - Added code for parsing Assign and Equal tokens.
aziz
parents: 25
diff changeset
513 else {
24
903f91163f23 - Added code for parsing AndAssign, AndLogical and AndBinary tokens.
aziz
parents: 23
diff changeset
514 t.type = TOK.AndBinary;
26
c3d7373db241 - Added code for parsing Assign and Equal tokens.
aziz
parents: 25
diff changeset
515 goto Lcommon2;
c3d7373db241 - Added code for parsing Assign and Equal tokens.
aziz
parents: 25
diff changeset
516 }
24
903f91163f23 - Added code for parsing AndAssign, AndLogical and AndBinary tokens.
aziz
parents: 23
diff changeset
517 goto Lcommon;
36
3c7210a722f7 - Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents: 35
diff changeset
518 case '+': /* + ++ += */
25
9c866208b3f6 - Added code for parsing PlusAssign, PlusPlus, Plus, MinusAssign, MinusMinus, Minus tokens.
aziz
parents: 24
diff changeset
519 c = *++p;
9c866208b3f6 - Added code for parsing PlusAssign, PlusPlus, Plus, MinusAssign, MinusMinus, Minus tokens.
aziz
parents: 24
diff changeset
520 if (c == '=')
9c866208b3f6 - Added code for parsing PlusAssign, PlusPlus, Plus, MinusAssign, MinusMinus, Minus tokens.
aziz
parents: 24
diff changeset
521 t.type = TOK.PlusAssign;
9c866208b3f6 - Added code for parsing PlusAssign, PlusPlus, Plus, MinusAssign, MinusMinus, Minus tokens.
aziz
parents: 24
diff changeset
522 else if (c == '+')
9c866208b3f6 - Added code for parsing PlusAssign, PlusPlus, Plus, MinusAssign, MinusMinus, Minus tokens.
aziz
parents: 24
diff changeset
523 t.type = TOK.PlusPlus;
26
c3d7373db241 - Added code for parsing Assign and Equal tokens.
aziz
parents: 25
diff changeset
524 else {
25
9c866208b3f6 - Added code for parsing PlusAssign, PlusPlus, Plus, MinusAssign, MinusMinus, Minus tokens.
aziz
parents: 24
diff changeset
525 t.type = TOK.Plus;
26
c3d7373db241 - Added code for parsing Assign and Equal tokens.
aziz
parents: 25
diff changeset
526 goto Lcommon2;
c3d7373db241 - Added code for parsing Assign and Equal tokens.
aziz
parents: 25
diff changeset
527 }
25
9c866208b3f6 - Added code for parsing PlusAssign, PlusPlus, Plus, MinusAssign, MinusMinus, Minus tokens.
aziz
parents: 24
diff changeset
528 goto Lcommon;
36
3c7210a722f7 - Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents: 35
diff changeset
529 case '-': /* - -- -= */
25
9c866208b3f6 - Added code for parsing PlusAssign, PlusPlus, Plus, MinusAssign, MinusMinus, Minus tokens.
aziz
parents: 24
diff changeset
530 c = *++p;
9c866208b3f6 - Added code for parsing PlusAssign, PlusPlus, Plus, MinusAssign, MinusMinus, Minus tokens.
aziz
parents: 24
diff changeset
531 if (c == '=')
9c866208b3f6 - Added code for parsing PlusAssign, PlusPlus, Plus, MinusAssign, MinusMinus, Minus tokens.
aziz
parents: 24
diff changeset
532 t.type = TOK.MinusAssign;
9c866208b3f6 - Added code for parsing PlusAssign, PlusPlus, Plus, MinusAssign, MinusMinus, Minus tokens.
aziz
parents: 24
diff changeset
533 else if (c == '-')
9c866208b3f6 - Added code for parsing PlusAssign, PlusPlus, Plus, MinusAssign, MinusMinus, Minus tokens.
aziz
parents: 24
diff changeset
534 t.type = TOK.MinusMinus;
26
c3d7373db241 - Added code for parsing Assign and Equal tokens.
aziz
parents: 25
diff changeset
535 else {
c3d7373db241 - Added code for parsing Assign and Equal tokens.
aziz
parents: 25
diff changeset
536 t.type = TOK.Minus;
c3d7373db241 - Added code for parsing Assign and Equal tokens.
aziz
parents: 25
diff changeset
537 goto Lcommon2;
c3d7373db241 - Added code for parsing Assign and Equal tokens.
aziz
parents: 25
diff changeset
538 }
c3d7373db241 - Added code for parsing Assign and Equal tokens.
aziz
parents: 25
diff changeset
539 goto Lcommon;
36
3c7210a722f7 - Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents: 35
diff changeset
540 case '=': /* = == */
26
c3d7373db241 - Added code for parsing Assign and Equal tokens.
aziz
parents: 25
diff changeset
541 if (p[1] == '=') {
c3d7373db241 - Added code for parsing Assign and Equal tokens.
aziz
parents: 25
diff changeset
542 ++p;
c3d7373db241 - Added code for parsing Assign and Equal tokens.
aziz
parents: 25
diff changeset
543 t.type = TOK.Equal;
c3d7373db241 - Added code for parsing Assign and Equal tokens.
aziz
parents: 25
diff changeset
544 }
25
9c866208b3f6 - Added code for parsing PlusAssign, PlusPlus, Plus, MinusAssign, MinusMinus, Minus tokens.
aziz
parents: 24
diff changeset
545 else
26
c3d7373db241 - Added code for parsing Assign and Equal tokens.
aziz
parents: 25
diff changeset
546 t.type = TOK.Assign;
25
9c866208b3f6 - Added code for parsing PlusAssign, PlusPlus, Plus, MinusAssign, MinusMinus, Minus tokens.
aziz
parents: 24
diff changeset
547 goto Lcommon;
36
3c7210a722f7 - Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents: 35
diff changeset
548 case '~': /* ~ ~= */
27
43b6bf56f0e9 - Added code for parsing CatAssign and Tilde tokens.
aziz
parents: 26
diff changeset
549 if (p[1] == '=') {
43b6bf56f0e9 - Added code for parsing CatAssign and Tilde tokens.
aziz
parents: 26
diff changeset
550 ++p;
43b6bf56f0e9 - Added code for parsing CatAssign and Tilde tokens.
aziz
parents: 26
diff changeset
551 t.type = TOK.CatAssign;
43b6bf56f0e9 - Added code for parsing CatAssign and Tilde tokens.
aziz
parents: 26
diff changeset
552 }
43b6bf56f0e9 - Added code for parsing CatAssign and Tilde tokens.
aziz
parents: 26
diff changeset
553 else
43b6bf56f0e9 - Added code for parsing CatAssign and Tilde tokens.
aziz
parents: 26
diff changeset
554 t.type = TOK.Tilde;
43b6bf56f0e9 - Added code for parsing CatAssign and Tilde tokens.
aziz
parents: 26
diff changeset
555 goto Lcommon;
36
3c7210a722f7 - Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents: 35
diff changeset
556 case '*': /* * *= */
29
ef83eea26bbd - Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents: 28
diff changeset
557 if (p[1] == '=') {
ef83eea26bbd - Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents: 28
diff changeset
558 ++p;
ef83eea26bbd - Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents: 28
diff changeset
559 t.type = TOK.MulAssign;
ef83eea26bbd - Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents: 28
diff changeset
560 }
ef83eea26bbd - Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents: 28
diff changeset
561 else
ef83eea26bbd - Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents: 28
diff changeset
562 t.type = TOK.Mul;
ef83eea26bbd - Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents: 28
diff changeset
563 goto Lcommon;
36
3c7210a722f7 - Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents: 35
diff changeset
564 case '^': /* ^ ^= */
29
ef83eea26bbd - Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents: 28
diff changeset
565 if (p[1] == '=') {
ef83eea26bbd - Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents: 28
diff changeset
566 ++p;
ef83eea26bbd - Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents: 28
diff changeset
567 t.type = TOK.XorAssign;
ef83eea26bbd - Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents: 28
diff changeset
568 }
ef83eea26bbd - Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents: 28
diff changeset
569 else
ef83eea26bbd - Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents: 28
diff changeset
570 t.type = TOK.Xor;
ef83eea26bbd - Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents: 28
diff changeset
571 goto Lcommon;
36
3c7210a722f7 - Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents: 35
diff changeset
572 case '%': /* % %= */
29
ef83eea26bbd - Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents: 28
diff changeset
573 if (p[1] == '=') {
ef83eea26bbd - Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents: 28
diff changeset
574 ++p;
ef83eea26bbd - Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents: 28
diff changeset
575 t.type = TOK.ModAssign;
ef83eea26bbd - Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents: 28
diff changeset
576 }
ef83eea26bbd - Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents: 28
diff changeset
577 else
ef83eea26bbd - Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents: 28
diff changeset
578 t.type = TOK.Mod;
ef83eea26bbd - Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents: 28
diff changeset
579 goto Lcommon;
ef83eea26bbd - Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents: 28
diff changeset
580 // Single character tokens:
20
d6adfbd7c513 - Added code for parsing braces.
aziz
parents: 19
diff changeset
581 case '(':
d6adfbd7c513 - Added code for parsing braces.
aziz
parents: 19
diff changeset
582 t.type = TOK.LParen;
d6adfbd7c513 - Added code for parsing braces.
aziz
parents: 19
diff changeset
583 goto Lcommon;
d6adfbd7c513 - Added code for parsing braces.
aziz
parents: 19
diff changeset
584 case ')':
d6adfbd7c513 - Added code for parsing braces.
aziz
parents: 19
diff changeset
585 t.type = TOK.RParen;
d6adfbd7c513 - Added code for parsing braces.
aziz
parents: 19
diff changeset
586 goto Lcommon;
d6adfbd7c513 - Added code for parsing braces.
aziz
parents: 19
diff changeset
587 case '[':
d6adfbd7c513 - Added code for parsing braces.
aziz
parents: 19
diff changeset
588 t.type = TOK.LBracket;
d6adfbd7c513 - Added code for parsing braces.
aziz
parents: 19
diff changeset
589 goto Lcommon;
d6adfbd7c513 - Added code for parsing braces.
aziz
parents: 19
diff changeset
590 case ']':
d6adfbd7c513 - Added code for parsing braces.
aziz
parents: 19
diff changeset
591 t.type = TOK.RBracket;
d6adfbd7c513 - Added code for parsing braces.
aziz
parents: 19
diff changeset
592 goto Lcommon;
d6adfbd7c513 - Added code for parsing braces.
aziz
parents: 19
diff changeset
593 case '{':
d6adfbd7c513 - Added code for parsing braces.
aziz
parents: 19
diff changeset
594 t.type = TOK.LBrace;
d6adfbd7c513 - Added code for parsing braces.
aziz
parents: 19
diff changeset
595 goto Lcommon;
d6adfbd7c513 - Added code for parsing braces.
aziz
parents: 19
diff changeset
596 case '}':
d6adfbd7c513 - Added code for parsing braces.
aziz
parents: 19
diff changeset
597 t.type = TOK.RBrace;
21
c785c122e4e6 - Added code for parsing Colon, Semicolon, Question, Comma and Dollar.
aziz
parents: 20
diff changeset
598 goto Lcommon;
c785c122e4e6 - Added code for parsing Colon, Semicolon, Question, Comma and Dollar.
aziz
parents: 20
diff changeset
599 case ':':
c785c122e4e6 - Added code for parsing Colon, Semicolon, Question, Comma and Dollar.
aziz
parents: 20
diff changeset
600 t.type = TOK.Colon;
c785c122e4e6 - Added code for parsing Colon, Semicolon, Question, Comma and Dollar.
aziz
parents: 20
diff changeset
601 goto Lcommon;
c785c122e4e6 - Added code for parsing Colon, Semicolon, Question, Comma and Dollar.
aziz
parents: 20
diff changeset
602 case ';':
c785c122e4e6 - Added code for parsing Colon, Semicolon, Question, Comma and Dollar.
aziz
parents: 20
diff changeset
603 t.type = TOK.Semicolon;
c785c122e4e6 - Added code for parsing Colon, Semicolon, Question, Comma and Dollar.
aziz
parents: 20
diff changeset
604 goto Lcommon;
c785c122e4e6 - Added code for parsing Colon, Semicolon, Question, Comma and Dollar.
aziz
parents: 20
diff changeset
605 case '?':
c785c122e4e6 - Added code for parsing Colon, Semicolon, Question, Comma and Dollar.
aziz
parents: 20
diff changeset
606 t.type = TOK.Question;
c785c122e4e6 - Added code for parsing Colon, Semicolon, Question, Comma and Dollar.
aziz
parents: 20
diff changeset
607 goto Lcommon;
c785c122e4e6 - Added code for parsing Colon, Semicolon, Question, Comma and Dollar.
aziz
parents: 20
diff changeset
608 case ',':
c785c122e4e6 - Added code for parsing Colon, Semicolon, Question, Comma and Dollar.
aziz
parents: 20
diff changeset
609 t.type = TOK.Comma;
c785c122e4e6 - Added code for parsing Colon, Semicolon, Question, Comma and Dollar.
aziz
parents: 20
diff changeset
610 goto Lcommon;
c785c122e4e6 - Added code for parsing Colon, Semicolon, Question, Comma and Dollar.
aziz
parents: 20
diff changeset
611 case '$':
c785c122e4e6 - Added code for parsing Colon, Semicolon, Question, Comma and Dollar.
aziz
parents: 20
diff changeset
612 t.type = TOK.Dollar;
20
d6adfbd7c513 - Added code for parsing braces.
aziz
parents: 19
diff changeset
613 Lcommon:
d6adfbd7c513 - Added code for parsing braces.
aziz
parents: 19
diff changeset
614 ++p;
26
c3d7373db241 - Added code for parsing Assign and Equal tokens.
aziz
parents: 25
diff changeset
615 Lcommon2:
20
d6adfbd7c513 - Added code for parsing braces.
aziz
parents: 19
diff changeset
616 t.end = p;
d6adfbd7c513 - Added code for parsing braces.
aziz
parents: 19
diff changeset
617 return;
30
426767b94635 - Added code for parsing the '#line' special token.
aziz
parents: 29
diff changeset
618 case '#':
360
b6a3755eba94 - Renamed scanSpecialToken() to scanSpecialTokenSequence().
aziz
parents: 350
diff changeset
619 return scanSpecialTokenSequence(t);
20
d6adfbd7c513 - Added code for parsing braces.
aziz
parents: 19
diff changeset
620 default:
d6adfbd7c513 - Added code for parsing braces.
aziz
parents: 19
diff changeset
621 }
d6adfbd7c513 - Added code for parsing braces.
aziz
parents: 19
diff changeset
622
411
cca83c0c00fd Added __EOF__ token.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 410
diff changeset
623 // Check for EOF
cca83c0c00fd Added __EOF__ token.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 410
diff changeset
624 if (c == 0 || c == _Z_)
cca83c0c00fd Added __EOF__ token.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 410
diff changeset
625 {
cca83c0c00fd Added __EOF__ token.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 410
diff changeset
626 assert(*p == 0 || *p == _Z_);
cca83c0c00fd Added __EOF__ token.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 410
diff changeset
627 t.type = TOK.EOF;
cca83c0c00fd Added __EOF__ token.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 410
diff changeset
628 t.end = p;
cca83c0c00fd Added __EOF__ token.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 410
diff changeset
629 tail = &t;
cca83c0c00fd Added __EOF__ token.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 410
diff changeset
630 assert(t.start == t.end);
cca83c0c00fd Added __EOF__ token.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 410
diff changeset
631 return;
cca83c0c00fd Added __EOF__ token.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 410
diff changeset
632 }
cca83c0c00fd Added __EOF__ token.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 410
diff changeset
633
390
4d36eea1bbc9 Refactored Lexer.scan().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 389
diff changeset
634 if (c & 128)
4d36eea1bbc9 Refactored Lexer.scan().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 389
diff changeset
635 {
4d36eea1bbc9 Refactored Lexer.scan().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 389
diff changeset
636 c = decodeUTF8();
4d36eea1bbc9 Refactored Lexer.scan().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 389
diff changeset
637 if (isUniAlpha(c))
4d36eea1bbc9 Refactored Lexer.scan().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 389
diff changeset
638 goto Lidentifier;
4d36eea1bbc9 Refactored Lexer.scan().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 389
diff changeset
639 }
4d36eea1bbc9 Refactored Lexer.scan().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 389
diff changeset
640
422
ad7977fe315a Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 420
diff changeset
641 error(t.start, MID.IllegalCharacter, cast(dchar)c);
390
4d36eea1bbc9 Refactored Lexer.scan().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 389
diff changeset
642
4d36eea1bbc9 Refactored Lexer.scan().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 389
diff changeset
643 ++p;
4d36eea1bbc9 Refactored Lexer.scan().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 389
diff changeset
644 t.type = TOK.Illegal;
4d36eea1bbc9 Refactored Lexer.scan().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 389
diff changeset
645 t.dchar_ = c;
4d36eea1bbc9 Refactored Lexer.scan().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 389
diff changeset
646 t.end = p;
4d36eea1bbc9 Refactored Lexer.scan().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 389
diff changeset
647 return;
4
92df59b1ec4a - Started implementation of scan().
aziz
parents: 3
diff changeset
648 }
3
4bbce78bfb1e - Added TOK enum.
aziz
parents: 2
diff changeset
649 }
4
92df59b1ec4a - Started implementation of scan().
aziz
parents: 3
diff changeset
650
413
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
651 template toUint(char[] T)
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
652 {
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
653 static assert(0 < T.length && T.length <= 4);
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
654 static if (T.length == 1)
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
655 const uint toUint = T[0];
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
656 else
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
657 const uint toUint = (T[0] << ((T.length-1)*8)) | toUint!(T[1..$]);
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
658 }
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
659 static assert(toUint!("\xAA\xBB\xCC\xDD") == 0xAABBCCDD);
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
660
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
661 // Can't use this yet due to a bug in DMD (bug id=1534).
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
662 template case_(char[] str, TOK tok, char[] label)
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
663 {
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
664 const char[] case_ =
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
665 `case `~toUint!(str).stringof~`:
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
666
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
667 goto `~label~`;`;
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
668 }
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
669
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
670 template case_L4(char[] str, TOK tok)
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
671 {
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
672 const char[] case_L4 = case_!(str, tok, "Lcommon_4");
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
673 }
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
674
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
675 template case_L3(char[] str, TOK tok)
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
676 {
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
677 const char[] case_L3 = case_!(str, tok, "Lcommon_3");
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
678 }
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
679
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
680 template case_L2(char[] str, TOK tok)
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
681 {
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
682 const char[] case_L2 = case_!(str, tok, "Lcommon_2");
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
683 }
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
684
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
685 template case_L1(char[] str, TOK tok)
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
686 {
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
687 const char[] case_L3 = case_!(str, tok, "Lcommon");
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
688 }
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
689
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
690 public void scan(out Token t)
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
691 in
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
692 {
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
693 assert(text.ptr <= p && p < end);
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
694 }
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
695 out
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
696 {
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
697 assert(text.ptr <= t.start && t.start < end, Token.toString(t.type));
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
698 assert(text.ptr <= t.end && t.end <= end, Token.toString(t.type));
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
699 }
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
700 body
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
701 {
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
702 // Scan whitespace.
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
703 auto pws = p;
418
7354f15cd5e9 Applied some fixes to the Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 416
diff changeset
704 auto old_loc = this.loc;
413
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
705 while (1)
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
706 {
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
707 switch (*p)
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
708 {
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
709 case '\r':
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
710 if (p[1] == '\n')
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
711 ++p;
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
712 case '\n':
419
89e40d43065d Added new member 'p_newl' to Lexer and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 418
diff changeset
713 assert(*p == '\n' || *p == '\r' || *p == LS[2] || *p == PS[2]);
413
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
714 ++p;
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
715 ++loc;
422
ad7977fe315a Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 420
diff changeset
716 setLineBegin(p);
413
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
717 continue;
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
718 case LS[0]:
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
719 if (p[1] == LS[1] && (p[2] == LS[2] || p[2] == PS[2]))
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
720 {
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
721 ++p; ++p;
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
722 goto case '\n';
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
723 }
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
724 // goto default;
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
725 default:
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
726 if (!isspace(*p))
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
727 break;
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
728 ++p;
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
729 continue;
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
730 }
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
731 break; // Exit loop.
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
732 }
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
733
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
734 if (p != pws)
418
7354f15cd5e9 Applied some fixes to the Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 416
diff changeset
735 {
413
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
736 t.ws = pws;
418
7354f15cd5e9 Applied some fixes to the Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 416
diff changeset
737 if (old_loc != this.loc)
7354f15cd5e9 Applied some fixes to the Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 416
diff changeset
738 version(token2LocTable)
420
ce644d724d87 Made Location a class and extended it with new members.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 419
diff changeset
739 token2LocTable[&t] = new Location(loc, null);
418
7354f15cd5e9 Applied some fixes to the Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 416
diff changeset
740 }
413
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
741
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
742 // Scan token.
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
743 t.start = p;
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
744
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
745 uint c = *p;
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
746 assert(end - p != 0);
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
747 switch (end - p)
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
748 {
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
749 case 1:
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
750 goto L1character;
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
751 case 2:
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
752 c <<= 8; c |= p[1];
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
753 goto L2characters;
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
754 case 3:
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
755 c <<= 8; c |= p[1]; c <<= 8; c |= p[2];
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
756 goto L3characters;
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
757 default:
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
758 version(BigEndian)
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
759 c = *cast(uint*)p;
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
760 else
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
761 {
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
762 c <<= 8; c |= p[1]; c <<= 8; c |= p[2]; c <<= 8; c |= p[3];
414
9c69615a4876 Added method insertEmptyTokenBefore() to Lexer. Made some fixes, too.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 413
diff changeset
763 /+
9c69615a4876 Added method insertEmptyTokenBefore() to Lexer. Made some fixes, too.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 413
diff changeset
764 c = *cast(uint*)p;
9c69615a4876 Added method insertEmptyTokenBefore() to Lexer. Made some fixes, too.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 413
diff changeset
765 asm
9c69615a4876 Added method insertEmptyTokenBefore() to Lexer. Made some fixes, too.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 413
diff changeset
766 {
9c69615a4876 Added method insertEmptyTokenBefore() to Lexer. Made some fixes, too.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 413
diff changeset
767 mov EDX, c;
9c69615a4876 Added method insertEmptyTokenBefore() to Lexer. Made some fixes, too.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 413
diff changeset
768 bswap EDX;
9c69615a4876 Added method insertEmptyTokenBefore() to Lexer. Made some fixes, too.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 413
diff changeset
769 mov c, EDX;
9c69615a4876 Added method insertEmptyTokenBefore() to Lexer. Made some fixes, too.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 413
diff changeset
770 }
9c69615a4876 Added method insertEmptyTokenBefore() to Lexer. Made some fixes, too.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 413
diff changeset
771 +/
413
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
772 }
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
773 }
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
774
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
775 // 4 character tokens.
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
776 switch (c)
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
777 {
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
778 case toUint!(">>>="):
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
779 t.type = TOK.RShiftAssign;
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
780 goto Lcommon_4;
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
781 case toUint!("!<>="):
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
782 t.type = TOK.Unordered;
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
783 Lcommon_4:
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
784 p += 4;
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
785 t.end = p;
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
786 return;
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
787 default:
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
788 }
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
789
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
790 c >>>= 8;
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
791 L3characters:
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
792 assert(p == t.start);
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
793 // 3 character tokens.
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
794 switch (c)
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
795 {
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
796 case toUint!(">>="):
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
797 t.type = TOK.RShiftAssign;
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
798 goto Lcommon_3;
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
799 case toUint!(">>>"):
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
800 t.type = TOK.URShift;
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
801 goto Lcommon_3;
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
802 case toUint!("<>="):
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
803 t.type = TOK.LorEorG;
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
804 goto Lcommon_3;
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
805 case toUint!("<<="):
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
806 t.type = TOK.LShiftAssign;
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
807 goto Lcommon_3;
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
808 case toUint!("!<="):
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
809 t.type = TOK.UorG;
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
810 goto Lcommon_3;
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
811 case toUint!("!>="):
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
812 t.type = TOK.UorL;
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
813 goto Lcommon_3;
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
814 case toUint!("!<>"):
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
815 t.type = TOK.UorE;
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
816 goto Lcommon_3;
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
817 case toUint!("..."):
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
818 t.type = TOK.Ellipses;
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
819 Lcommon_3:
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
820 p += 3;
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
821 t.end = p;
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
822 return;
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
823 default:
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
824 }
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
825
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
826 c >>>= 8;
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
827 L2characters:
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
828 assert(p == t.start);
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
829 // 2 character tokens.
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
830 switch (c)
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
831 {
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
832 case toUint!("/+"):
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
833 ++p; // Skip /
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
834 return scanNestedComment(t);
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
835 case toUint!("/*"):
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
836 ++p; // Skip /
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
837 return scanBlockComment(t);
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
838 case toUint!("//"):
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
839 ++p; // Skip /
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
840 assert(*p == '/');
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
841 while (1)
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
842 {
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
843 c = *++p;
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
844 switch (c)
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
845 {
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
846 case '\r', '\n', 0, _Z_:
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
847 break;
419
89e40d43065d Added new member 'p_newl' to Lexer and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 418
diff changeset
848 case LS[0]:
89e40d43065d Added new member 'p_newl' to Lexer and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 418
diff changeset
849 if (p[1] == LS[1] && (p[2] == LS[2] || p[2] == PS[2]))
89e40d43065d Added new member 'p_newl' to Lexer and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 418
diff changeset
850 break;
413
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
851 default:
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
852 if (c & 128)
419
89e40d43065d Added new member 'p_newl' to Lexer and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 418
diff changeset
853 decodeUTF8();
413
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
854 continue;
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
855 }
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
856 break; // Exit loop.
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
857 }
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
858 t.type = TOK.Comment;
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
859 t.end = p;
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
860 return;
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
861 case toUint!(">="):
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
862 t.type = TOK.GreaterEqual;
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
863 goto Lcommon_2;
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
864 case toUint!(">>"):
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
865 t.type = TOK.RShift;
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
866 goto Lcommon_2;
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
867 case toUint!("<<"):
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
868 t.type = TOK.LShift;
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
869 goto Lcommon_2;
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
870 case toUint!("<="):
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
871 t.type = TOK.LessEqual;
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
872 goto Lcommon_2;
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
873 case toUint!("<>"):
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
874 t.type = TOK.LorG;
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
875 goto Lcommon_2;
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
876 case toUint!("!<"):
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
877 t.type = TOK.UorGorE;
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
878 goto Lcommon_2;
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
879 case toUint!("!>"):
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
880 t.type = TOK.UorLorE;
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
881 goto Lcommon_2;
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
882 case toUint!("!="):
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
883 t.type = TOK.NotEqual;
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
884 goto Lcommon_2;
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
885 case toUint!(".."):
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
886 t.type = TOK.Slice;
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
887 goto Lcommon_2;
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
888 case toUint!("&&"):
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
889 t.type = TOK.AndLogical;
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
890 goto Lcommon_2;
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
891 case toUint!("&="):
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
892 t.type = TOK.AndAssign;
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
893 goto Lcommon_2;
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
894 case toUint!("||"):
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
895 t.type = TOK.OrLogical;
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
896 goto Lcommon_2;
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
897 case toUint!("|="):
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
898 t.type = TOK.OrAssign;
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
899 goto Lcommon_2;
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
900 case toUint!("++"):
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
901 t.type = TOK.PlusPlus;
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
902 goto Lcommon_2;
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
903 case toUint!("+="):
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
904 t.type = TOK.PlusAssign;
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
905 goto Lcommon_2;
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
906 case toUint!("--"):
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
907 t.type = TOK.MinusMinus;
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
908 goto Lcommon_2;
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
909 case toUint!("-="):
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
910 t.type = TOK.MinusAssign;
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
911 goto Lcommon_2;
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
912 case toUint!("=="):
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
913 t.type = TOK.Equal;
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
914 goto Lcommon_2;
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
915 case toUint!("~="):
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
916 t.type = TOK.CatAssign;
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
917 goto Lcommon_2;
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
918 case toUint!("*="):
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
919 t.type = TOK.MulAssign;
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
920 goto Lcommon_2;
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
921 case toUint!("/="):
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
922 t.type = TOK.DivAssign;
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
923 goto Lcommon_2;
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
924 case toUint!("^="):
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
925 t.type = TOK.XorAssign;
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
926 goto Lcommon_2;
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
927 case toUint!("%="):
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
928 t.type = TOK.ModAssign;
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
929 Lcommon_2:
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
930 p += 2;
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
931 t.end = p;
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
932 return;
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
933 default:
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
934 }
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
935
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
936 c >>>= 8;
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
937 L1character:
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
938 assert(p == t.start);
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
939 assert(*p == c, Format("p={0},c={1}", *p, cast(dchar)c));
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
940 // 1 character tokens.
414
9c69615a4876 Added method insertEmptyTokenBefore() to Lexer. Made some fixes, too.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 413
diff changeset
941 // TODO: consider storing the token type in ptable.
413
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
942 switch (c)
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
943 {
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
944 case '\'':
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
945 return scanCharacterLiteral(t);
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
946 case '`':
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
947 return scanRawStringLiteral(t);
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
948 case '"':
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
949 return scanNormalStringLiteral(t);
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
950 case '\\':
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
951 char[] buffer;
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
952 do
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
953 {
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
954 c = scanEscapeSequence();
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
955 if (c < 128)
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
956 buffer ~= c;
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
957 else
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
958 encodeUTF8(buffer, c);
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
959 } while (*p == '\\')
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
960 buffer ~= 0;
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
961 t.type = TOK.String;
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
962 t.str = buffer;
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
963 t.end = p;
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
964 return;
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
965 case '<':
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
966 t.type = TOK.Greater;
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
967 goto Lcommon;
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
968 case '>':
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
969 t.type = TOK.Less;
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
970 goto Lcommon;
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
971 case '^':
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
972 t.type = TOK.Xor;
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
973 goto Lcommon;
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
974 case '!':
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
975 t.type = TOK.Not;
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
976 goto Lcommon;
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
977 case '.':
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
978 if (isdigit(p[1]))
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
979 return scanReal(t);
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
980 t.type = TOK.Dot;
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
981 goto Lcommon;
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
982 case '&':
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
983 t.type = TOK.AndBinary;
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
984 goto Lcommon;
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
985 case '|':
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
986 t.type = TOK.OrBinary;
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
987 goto Lcommon;
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
988 case '+':
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
989 t.type = TOK.Plus;
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
990 goto Lcommon;
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
991 case '-':
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
992 t.type = TOK.Minus;
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
993 goto Lcommon;
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
994 case '=':
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
995 t.type = TOK.Assign;
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
996 goto Lcommon;
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
997 case '~':
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
998 t.type = TOK.Tilde;
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
999 goto Lcommon;
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
1000 case '*':
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
1001 t.type = TOK.Mul;
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
1002 goto Lcommon;
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
1003 case '/':
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
1004 t.type = TOK.Div;
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
1005 goto Lcommon;
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
1006 case '%':
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
1007 t.type = TOK.Mod;
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
1008 goto Lcommon;
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
1009 case '(':
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
1010 t.type = TOK.LParen;
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
1011 goto Lcommon;
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
1012 case ')':
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
1013 t.type = TOK.RParen;
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
1014 goto Lcommon;
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
1015 case '[':
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
1016 t.type = TOK.LBracket;
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
1017 goto Lcommon;
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
1018 case ']':
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
1019 t.type = TOK.RBracket;
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
1020 goto Lcommon;
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
1021 case '{':
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
1022 t.type = TOK.LBrace;
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
1023 goto Lcommon;
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
1024 case '}':
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
1025 t.type = TOK.RBrace;
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
1026 goto Lcommon;
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
1027 case ':':
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
1028 t.type = TOK.Colon;
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
1029 goto Lcommon;
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
1030 case ';':
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
1031 t.type = TOK.Semicolon;
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
1032 goto Lcommon;
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
1033 case '?':
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
1034 t.type = TOK.Question;
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
1035 goto Lcommon;
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
1036 case ',':
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
1037 t.type = TOK.Comma;
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
1038 goto Lcommon;
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
1039 case '$':
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
1040 t.type = TOK.Dollar;
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
1041 Lcommon:
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
1042 ++p;
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
1043 t.end = p;
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
1044 return;
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
1045 case '#':
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
1046 return scanSpecialTokenSequence(t);
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
1047 default:
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
1048 }
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
1049
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
1050 assert(p == t.start);
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
1051 assert(*p == c);
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
1052
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
1053 // TODO: consider moving isidbeg() and isdigit() up.
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
1054 if (isidbeg(c))
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
1055 {
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
1056 if (c == 'r' && p[1] == '"' && ++p)
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
1057 return scanRawStringLiteral(t);
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
1058 if (c == 'x' && p[1] == '"')
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
1059 return scanHexStringLiteral(t);
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
1060 version(D2)
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
1061 {
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
1062 if (c == 'q' && p[1] == '"')
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
1063 return scanDelimitedStringLiteral(t);
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
1064 if (c == 'q' && p[1] == '{')
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
1065 return scanTokenStringLiteral(t);
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
1066 }
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
1067 Lidentifier:
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
1068 do
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
1069 { c = *++p; }
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
1070 while (isident(c) || c & 128 && isUniAlpha(decodeUTF8()))
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
1071
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
1072 t.end = p;
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
1073
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
1074 string str = t.srcText;
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
1075 Identifier* id = str in idtable;
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
1076
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
1077 if (!id)
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
1078 {
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
1079 idtable[str] = Identifier(TOK.Identifier, str);
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
1080 id = str in idtable;
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
1081 }
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
1082 assert(id);
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
1083 t.type = id.type;
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
1084 if (t.type == TOK.Identifier)
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
1085 return;
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
1086 if (t.type == TOK.EOF)
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
1087 {
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
1088 t.type = TOK.EOF;
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
1089 t.end = p;
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
1090 tail = &t;
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
1091 assert(t.srcText == "__EOF__");
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
1092 }
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
1093 else if (t.isSpecialToken)
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
1094 finalizeSpecialToken(t);
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
1095 return;
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
1096 }
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
1097
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
1098 if (isdigit(c))
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
1099 return scanNumber(t);
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
1100
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
1101 // Check for EOF
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
1102 if (c == 0 || c == _Z_)
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
1103 {
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
1104 assert(*p == 0 || *p == _Z_, *p~"");
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
1105 t.type = TOK.EOF;
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
1106 t.end = p;
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
1107 tail = &t;
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
1108 assert(t.start == t.end);
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
1109 return;
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
1110 }
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
1111
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
1112 if (c & 128)
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
1113 {
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
1114 c = decodeUTF8();
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
1115 if (isUniAlpha(c))
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
1116 goto Lidentifier;
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
1117 }
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
1118
422
ad7977fe315a Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 420
diff changeset
1119 error(t.start, MID.IllegalCharacter, cast(dchar)c);
413
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
1120
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
1121 ++p;
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
1122 t.type = TOK.Illegal;
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
1123 t.dchar_ = c;
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
1124 t.end = p;
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
1125 return;
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
1126 }
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
1127
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
1128 void scanBlockComment(ref Token t)
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
1129 {
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
1130 assert(p[-1] == '/' && *p == '*');
422
ad7977fe315a Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 420
diff changeset
1131 auto tokenLineNum = loc;
ad7977fe315a Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 420
diff changeset
1132 auto tokenLineBegin = lineBegin;
413
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
1133 uint c;
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
1134 while (1)
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
1135 {
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
1136 c = *++p;
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
1137 LswitchBC: // only jumped to from default case of next switch(c)
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
1138 switch (c)
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
1139 {
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
1140 case '\r':
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
1141 if (p[1] == '\n')
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
1142 ++p;
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
1143 case '\n':
419
89e40d43065d Added new member 'p_newl' to Lexer and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 418
diff changeset
1144 assert(*p == '\n' || *p == '\r' || *p == LS[2] || *p == PS[2]);
413
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
1145 ++loc;
422
ad7977fe315a Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 420
diff changeset
1146 setLineBegin(p+1);
413
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
1147 continue;
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
1148 case 0, _Z_:
422
ad7977fe315a Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 420
diff changeset
1149 error(tokenLineNum, tokenLineBegin, t.start, MID.UnterminatedBlockComment);
413
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
1150 goto LreturnBC;
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
1151 default:
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
1152 if (c & 128)
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
1153 {
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
1154 c = decodeUTF8();
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
1155 if (c == LSd || c == PSd)
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
1156 goto case '\n';
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
1157 continue;
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
1158 }
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
1159 }
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
1160
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
1161 c <<= 8;
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
1162 c |= *++p;
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
1163 switch (c)
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
1164 {
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
1165 case toUint!("*/"):
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
1166 ++p;
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
1167 LreturnBC:
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
1168 t.type = TOK.Comment;
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
1169 t.end = p;
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
1170 return;
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
1171 default:
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
1172 c &= char.max;
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
1173 goto LswitchBC;
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
1174 }
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
1175 }
422
ad7977fe315a Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 420
diff changeset
1176 assert(0);
413
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
1177 }
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
1178
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
1179 void scanNestedComment(ref Token t)
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
1180 {
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
1181 assert(p[-1] == '/' && *p == '+');
422
ad7977fe315a Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 420
diff changeset
1182 auto tokenLineNum = loc;
ad7977fe315a Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 420
diff changeset
1183 auto tokenLineBegin = lineBegin;
413
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
1184 uint level = 1;
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
1185 uint c;
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
1186 while (1)
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
1187 {
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
1188 c = *++p;
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
1189 LswitchNC: // only jumped to from default case of next switch(c)
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
1190 switch (c)
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
1191 {
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
1192 case '\r':
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
1193 if (p[1] == '\n')
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
1194 ++p;
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
1195 case '\n':
419
89e40d43065d Added new member 'p_newl' to Lexer and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 418
diff changeset
1196 assert(*p == '\n' || *p == '\r' || *p == LS[2] || *p == PS[2]);
413
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
1197 ++loc;
422
ad7977fe315a Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 420
diff changeset
1198 setLineBegin(p+1);
413
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
1199 continue;
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
1200 case 0, _Z_:
422
ad7977fe315a Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 420
diff changeset
1201 error(tokenLineNum, tokenLineBegin, t.start, MID.UnterminatedNestedComment);
413
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
1202 goto LreturnNC;
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
1203 default:
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
1204 if (c & 128)
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
1205 {
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
1206 c = decodeUTF8();
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
1207 if (c == LSd || c == PSd)
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
1208 goto case '\n';
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
1209 continue;
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
1210 }
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
1211 }
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
1212
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
1213 c <<= 8;
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
1214 c |= *++p;
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
1215 switch (c)
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
1216 {
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
1217 case toUint!("/+"):
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
1218 ++level;
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
1219 continue;
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
1220 case toUint!("+/"):
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
1221 if (--level == 0)
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
1222 {
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
1223 ++p;
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
1224 LreturnNC:
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
1225 t.type = TOK.Comment;
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
1226 t.end = p;
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
1227 return;
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
1228 }
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
1229 continue;
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
1230 default:
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
1231 c &= char.max;
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
1232 goto LswitchNC;
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
1233 }
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
1234 }
422
ad7977fe315a Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 420
diff changeset
1235 assert(0);
413
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
1236 }
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
1237
47
8aa37a78937b - Properly implemented scanner for normal string literals.
aziz
parents: 46
diff changeset
1238 void scanNormalStringLiteral(ref Token t)
8aa37a78937b - Properly implemented scanner for normal string literals.
aziz
parents: 46
diff changeset
1239 {
8aa37a78937b - Properly implemented scanner for normal string literals.
aziz
parents: 46
diff changeset
1240 assert(*p == '"');
422
ad7977fe315a Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 420
diff changeset
1241 auto tokenLineNum = loc;
ad7977fe315a Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 420
diff changeset
1242 auto tokenLineBegin = lineBegin;
47
8aa37a78937b - Properly implemented scanner for normal string literals.
aziz
parents: 46
diff changeset
1243 char[] buffer;
8aa37a78937b - Properly implemented scanner for normal string literals.
aziz
parents: 46
diff changeset
1244 t.type = TOK.String;
384
f7ce725e79c3 Refactored scanNormalStringLiteral().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 383
diff changeset
1245 uint c;
47
8aa37a78937b - Properly implemented scanner for normal string literals.
aziz
parents: 46
diff changeset
1246 while (1)
8aa37a78937b - Properly implemented scanner for normal string literals.
aziz
parents: 46
diff changeset
1247 {
384
f7ce725e79c3 Refactored scanNormalStringLiteral().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 383
diff changeset
1248 c = *++p;
f7ce725e79c3 Refactored scanNormalStringLiteral().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 383
diff changeset
1249 switch (c)
47
8aa37a78937b - Properly implemented scanner for normal string literals.
aziz
parents: 46
diff changeset
1250 {
8aa37a78937b - Properly implemented scanner for normal string literals.
aziz
parents: 46
diff changeset
1251 case '"':
8aa37a78937b - Properly implemented scanner for normal string literals.
aziz
parents: 46
diff changeset
1252 ++p;
8aa37a78937b - Properly implemented scanner for normal string literals.
aziz
parents: 46
diff changeset
1253 Lreturn:
8aa37a78937b - Properly implemented scanner for normal string literals.
aziz
parents: 46
diff changeset
1254 buffer ~= 0;
49
7f0fa15dcffc - Renamed function.
aziz
parents: 48
diff changeset
1255 t.str = buffer;
47
8aa37a78937b - Properly implemented scanner for normal string literals.
aziz
parents: 46
diff changeset
1256 t.pf = scanPostfix();
8aa37a78937b - Properly implemented scanner for normal string literals.
aziz
parents: 46
diff changeset
1257 t.end = p;
8aa37a78937b - Properly implemented scanner for normal string literals.
aziz
parents: 46
diff changeset
1258 return;
8aa37a78937b - Properly implemented scanner for normal string literals.
aziz
parents: 46
diff changeset
1259 case '\\':
384
f7ce725e79c3 Refactored scanNormalStringLiteral().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 383
diff changeset
1260 c = scanEscapeSequence();
f7ce725e79c3 Refactored scanNormalStringLiteral().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 383
diff changeset
1261 --p;
424
bb3cb00feeb2 Applied some fixes to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 422
diff changeset
1262 if (c < 128)
384
f7ce725e79c3 Refactored scanNormalStringLiteral().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 383
diff changeset
1263 break;
424
bb3cb00feeb2 Applied some fixes to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 422
diff changeset
1264 encodeUTF8(buffer, c);
47
8aa37a78937b - Properly implemented scanner for normal string literals.
aziz
parents: 46
diff changeset
1265 continue;
8aa37a78937b - Properly implemented scanner for normal string literals.
aziz
parents: 46
diff changeset
1266 case '\r':
8aa37a78937b - Properly implemented scanner for normal string literals.
aziz
parents: 46
diff changeset
1267 if (p[1] == '\n')
8aa37a78937b - Properly implemented scanner for normal string literals.
aziz
parents: 46
diff changeset
1268 ++p;
8aa37a78937b - Properly implemented scanner for normal string literals.
aziz
parents: 46
diff changeset
1269 case '\n':
419
89e40d43065d Added new member 'p_newl' to Lexer and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 418
diff changeset
1270 assert(*p == '\n' || *p == '\r' || *p == LS[2] || *p == PS[2]);
47
8aa37a78937b - Properly implemented scanner for normal string literals.
aziz
parents: 46
diff changeset
1271 ++loc;
384
f7ce725e79c3 Refactored scanNormalStringLiteral().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 383
diff changeset
1272 c = '\n'; // Convert EndOfLine to \n.
422
ad7977fe315a Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 420
diff changeset
1273 setLineBegin(p+1);
384
f7ce725e79c3 Refactored scanNormalStringLiteral().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 383
diff changeset
1274 break;
47
8aa37a78937b - Properly implemented scanner for normal string literals.
aziz
parents: 46
diff changeset
1275 case 0, _Z_:
422
ad7977fe315a Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 420
diff changeset
1276 error(tokenLineNum, tokenLineBegin, t.start, MID.UnterminatedString);
47
8aa37a78937b - Properly implemented scanner for normal string literals.
aziz
parents: 46
diff changeset
1277 goto Lreturn;
8aa37a78937b - Properly implemented scanner for normal string literals.
aziz
parents: 46
diff changeset
1278 default:
384
f7ce725e79c3 Refactored scanNormalStringLiteral().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 383
diff changeset
1279 if (c & 128)
47
8aa37a78937b - Properly implemented scanner for normal string literals.
aziz
parents: 46
diff changeset
1280 {
384
f7ce725e79c3 Refactored scanNormalStringLiteral().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 383
diff changeset
1281 c = decodeUTF8();
f7ce725e79c3 Refactored scanNormalStringLiteral().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 383
diff changeset
1282 if (c == LSd || c == PSd)
47
8aa37a78937b - Properly implemented scanner for normal string literals.
aziz
parents: 46
diff changeset
1283 goto case '\n';
49
7f0fa15dcffc - Renamed function.
aziz
parents: 48
diff changeset
1284
384
f7ce725e79c3 Refactored scanNormalStringLiteral().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 383
diff changeset
1285 encodeUTF8(buffer, c);
47
8aa37a78937b - Properly implemented scanner for normal string literals.
aziz
parents: 46
diff changeset
1286 continue;
8aa37a78937b - Properly implemented scanner for normal string literals.
aziz
parents: 46
diff changeset
1287 }
8aa37a78937b - Properly implemented scanner for normal string literals.
aziz
parents: 46
diff changeset
1288 }
424
bb3cb00feeb2 Applied some fixes to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 422
diff changeset
1289 assert(isascii(c));
384
f7ce725e79c3 Refactored scanNormalStringLiteral().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 383
diff changeset
1290 buffer ~= c;
47
8aa37a78937b - Properly implemented scanner for normal string literals.
aziz
parents: 46
diff changeset
1291 }
67
996065105910 - Fix: variadic arguments are local to the variadic function. Parameters are converted to an array of strings first and then passed to the constructor of class Problem.
aziz
parents: 66
diff changeset
1292 assert(0);
47
8aa37a78937b - Properly implemented scanner for normal string literals.
aziz
parents: 46
diff changeset
1293 }
8aa37a78937b - Properly implemented scanner for normal string literals.
aziz
parents: 46
diff changeset
1294
18
c48d2125f1e2 - Moved code for scanning character literals to separate function.
aziz
parents: 17
diff changeset
1295 void scanCharacterLiteral(ref Token t)
c48d2125f1e2 - Moved code for scanning character literals to separate function.
aziz
parents: 17
diff changeset
1296 {
c48d2125f1e2 - Moved code for scanning character literals to separate function.
aziz
parents: 17
diff changeset
1297 assert(*p == '\'');
c48d2125f1e2 - Moved code for scanning character literals to separate function.
aziz
parents: 17
diff changeset
1298 MID id = MID.UnterminatedCharacterLiteral;
46
e0dbd4722f74 - Scanning character literals correctly now.
aziz
parents: 45
diff changeset
1299 ++p;
82
fc645fb2fe72 - scanEscapeSequences() doesn't return 0xFFFF as an error value anymore, because it is a valid codepoint usable by the user.
aziz
parents: 71
diff changeset
1300 TOK type = TOK.CharLiteral;
46
e0dbd4722f74 - Scanning character literals correctly now.
aziz
parents: 45
diff changeset
1301 switch (*p)
18
c48d2125f1e2 - Moved code for scanning character literals to separate function.
aziz
parents: 17
diff changeset
1302 {
c48d2125f1e2 - Moved code for scanning character literals to separate function.
aziz
parents: 17
diff changeset
1303 case '\\':
386
392a0068fc61 Refactored code related to scanning escape sequences.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 384
diff changeset
1304 switch (p[1])
82
fc645fb2fe72 - scanEscapeSequences() doesn't return 0xFFFF as an error value anymore, because it is a valid codepoint usable by the user.
aziz
parents: 71
diff changeset
1305 {
fc645fb2fe72 - scanEscapeSequences() doesn't return 0xFFFF as an error value anymore, because it is a valid codepoint usable by the user.
aziz
parents: 71
diff changeset
1306 case 'u':
fc645fb2fe72 - scanEscapeSequences() doesn't return 0xFFFF as an error value anymore, because it is a valid codepoint usable by the user.
aziz
parents: 71
diff changeset
1307 type = TOK.WCharLiteral; break;
fc645fb2fe72 - scanEscapeSequences() doesn't return 0xFFFF as an error value anymore, because it is a valid codepoint usable by the user.
aziz
parents: 71
diff changeset
1308 case 'U':
fc645fb2fe72 - scanEscapeSequences() doesn't return 0xFFFF as an error value anymore, because it is a valid codepoint usable by the user.
aziz
parents: 71
diff changeset
1309 type = TOK.DCharLiteral; break;
fc645fb2fe72 - scanEscapeSequences() doesn't return 0xFFFF as an error value anymore, because it is a valid codepoint usable by the user.
aziz
parents: 71
diff changeset
1310 default:
fc645fb2fe72 - scanEscapeSequences() doesn't return 0xFFFF as an error value anymore, because it is a valid codepoint usable by the user.
aziz
parents: 71
diff changeset
1311 }
46
e0dbd4722f74 - Scanning character literals correctly now.
aziz
parents: 45
diff changeset
1312 t.dchar_ = scanEscapeSequence();
18
c48d2125f1e2 - Moved code for scanning character literals to separate function.
aziz
parents: 17
diff changeset
1313 break;
c48d2125f1e2 - Moved code for scanning character literals to separate function.
aziz
parents: 17
diff changeset
1314 case '\'':
46
e0dbd4722f74 - Scanning character literals correctly now.
aziz
parents: 45
diff changeset
1315 ++p;
18
c48d2125f1e2 - Moved code for scanning character literals to separate function.
aziz
parents: 17
diff changeset
1316 id = MID.EmptyCharacterLiteral;
419
89e40d43065d Added new member 'p_newl' to Lexer and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 418
diff changeset
1317 // fall through
46
e0dbd4722f74 - Scanning character literals correctly now.
aziz
parents: 45
diff changeset
1318 case '\n', '\r', 0, _Z_:
18
c48d2125f1e2 - Moved code for scanning character literals to separate function.
aziz
parents: 17
diff changeset
1319 goto Lerr;
419
89e40d43065d Added new member 'p_newl' to Lexer and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 418
diff changeset
1320 case LS[0]:
89e40d43065d Added new member 'p_newl' to Lexer and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 418
diff changeset
1321 if (p[1] == LS[1] && (p[2] == LS[2] || p[2] == PS[2]))
89e40d43065d Added new member 'p_newl' to Lexer and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 418
diff changeset
1322 goto Lerr;
89e40d43065d Added new member 'p_newl' to Lexer and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 418
diff changeset
1323 // fall through
18
c48d2125f1e2 - Moved code for scanning character literals to separate function.
aziz
parents: 17
diff changeset
1324 default:
46
e0dbd4722f74 - Scanning character literals correctly now.
aziz
parents: 45
diff changeset
1325 uint c = *p;
18
c48d2125f1e2 - Moved code for scanning character literals to separate function.
aziz
parents: 17
diff changeset
1326 if (c & 128)
c48d2125f1e2 - Moved code for scanning character literals to separate function.
aziz
parents: 17
diff changeset
1327 {
49
7f0fa15dcffc - Renamed function.
aziz
parents: 48
diff changeset
1328 c = decodeUTF8();
82
fc645fb2fe72 - scanEscapeSequences() doesn't return 0xFFFF as an error value anymore, because it is a valid codepoint usable by the user.
aziz
parents: 71
diff changeset
1329 if (c <= 0xFFFF)
fc645fb2fe72 - scanEscapeSequences() doesn't return 0xFFFF as an error value anymore, because it is a valid codepoint usable by the user.
aziz
parents: 71
diff changeset
1330 type = TOK.WCharLiteral;
fc645fb2fe72 - scanEscapeSequences() doesn't return 0xFFFF as an error value anymore, because it is a valid codepoint usable by the user.
aziz
parents: 71
diff changeset
1331 else
fc645fb2fe72 - scanEscapeSequences() doesn't return 0xFFFF as an error value anymore, because it is a valid codepoint usable by the user.
aziz
parents: 71
diff changeset
1332 type = TOK.DCharLiteral;
18
c48d2125f1e2 - Moved code for scanning character literals to separate function.
aziz
parents: 17
diff changeset
1333 }
46
e0dbd4722f74 - Scanning character literals correctly now.
aziz
parents: 45
diff changeset
1334 t.dchar_ = c;
e0dbd4722f74 - Scanning character literals correctly now.
aziz
parents: 45
diff changeset
1335 ++p;
18
c48d2125f1e2 - Moved code for scanning character literals to separate function.
aziz
parents: 17
diff changeset
1336 }
c48d2125f1e2 - Moved code for scanning character literals to separate function.
aziz
parents: 17
diff changeset
1337
46
e0dbd4722f74 - Scanning character literals correctly now.
aziz
parents: 45
diff changeset
1338 if (*p == '\'')
e0dbd4722f74 - Scanning character literals correctly now.
aziz
parents: 45
diff changeset
1339 ++p;
e0dbd4722f74 - Scanning character literals correctly now.
aziz
parents: 45
diff changeset
1340 else
82
fc645fb2fe72 - scanEscapeSequences() doesn't return 0xFFFF as an error value anymore, because it is a valid codepoint usable by the user.
aziz
parents: 71
diff changeset
1341 Lerr:
422
ad7977fe315a Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 420
diff changeset
1342 error(t.start, id);
82
fc645fb2fe72 - scanEscapeSequences() doesn't return 0xFFFF as an error value anymore, because it is a valid codepoint usable by the user.
aziz
parents: 71
diff changeset
1343 t.type = type;
18
c48d2125f1e2 - Moved code for scanning character literals to separate function.
aziz
parents: 17
diff changeset
1344 t.end = p;
c48d2125f1e2 - Moved code for scanning character literals to separate function.
aziz
parents: 17
diff changeset
1345 }
c48d2125f1e2 - Moved code for scanning character literals to separate function.
aziz
parents: 17
diff changeset
1346
33
cf3047cf3cd2 - Added code for parsing back quote and raw strings.
aziz
parents: 32
diff changeset
1347 char scanPostfix()
31
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix.
aziz
parents: 30
diff changeset
1348 {
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix.
aziz
parents: 30
diff changeset
1349 switch (*p)
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix.
aziz
parents: 30
diff changeset
1350 {
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix.
aziz
parents: 30
diff changeset
1351 case 'c':
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix.
aziz
parents: 30
diff changeset
1352 case 'w':
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix.
aziz
parents: 30
diff changeset
1353 case 'd':
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix.
aziz
parents: 30
diff changeset
1354 return *p++;
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix.
aziz
parents: 30
diff changeset
1355 default:
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix.
aziz
parents: 30
diff changeset
1356 return 0;
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix.
aziz
parents: 30
diff changeset
1357 }
112
004d98df65af - Implemented parseInterfaceDeclaration().
aziz
parents: 105
diff changeset
1358 assert(0);
31
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix.
aziz
parents: 30
diff changeset
1359 }
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix.
aziz
parents: 30
diff changeset
1360
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix.
aziz
parents: 30
diff changeset
1361 void scanRawStringLiteral(ref Token t)
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix.
aziz
parents: 30
diff changeset
1362 {
422
ad7977fe315a Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 420
diff changeset
1363 auto tokenLineNum = loc;
ad7977fe315a Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 420
diff changeset
1364 auto tokenLineBegin = lineBegin;
33
cf3047cf3cd2 - Added code for parsing back quote and raw strings.
aziz
parents: 32
diff changeset
1365 uint delim = *p;
cf3047cf3cd2 - Added code for parsing back quote and raw strings.
aziz
parents: 32
diff changeset
1366 assert(delim == '`' || delim == '"' && p[-1] == 'r');
cf3047cf3cd2 - Added code for parsing back quote and raw strings.
aziz
parents: 32
diff changeset
1367 t.type = TOK.String;
cf3047cf3cd2 - Added code for parsing back quote and raw strings.
aziz
parents: 32
diff changeset
1368 char[] buffer;
cf3047cf3cd2 - Added code for parsing back quote and raw strings.
aziz
parents: 32
diff changeset
1369 uint c;
cf3047cf3cd2 - Added code for parsing back quote and raw strings.
aziz
parents: 32
diff changeset
1370 while (1)
cf3047cf3cd2 - Added code for parsing back quote and raw strings.
aziz
parents: 32
diff changeset
1371 {
cf3047cf3cd2 - Added code for parsing back quote and raw strings.
aziz
parents: 32
diff changeset
1372 c = *++p;
cf3047cf3cd2 - Added code for parsing back quote and raw strings.
aziz
parents: 32
diff changeset
1373 switch (c)
cf3047cf3cd2 - Added code for parsing back quote and raw strings.
aziz
parents: 32
diff changeset
1374 {
cf3047cf3cd2 - Added code for parsing back quote and raw strings.
aziz
parents: 32
diff changeset
1375 case '\r':
cf3047cf3cd2 - Added code for parsing back quote and raw strings.
aziz
parents: 32
diff changeset
1376 if (p[1] == '\n')
cf3047cf3cd2 - Added code for parsing back quote and raw strings.
aziz
parents: 32
diff changeset
1377 ++p;
cf3047cf3cd2 - Added code for parsing back quote and raw strings.
aziz
parents: 32
diff changeset
1378 case '\n':
419
89e40d43065d Added new member 'p_newl' to Lexer and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 418
diff changeset
1379 assert(*p == '\n' || *p == '\r' || *p == LS[2] || *p == PS[2]);
383
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1380 c = '\n'; // Convert EndOfLine ('\r','\r\n','\n',LS,PS) to '\n'
33
cf3047cf3cd2 - Added code for parsing back quote and raw strings.
aziz
parents: 32
diff changeset
1381 ++loc;
422
ad7977fe315a Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 420
diff changeset
1382 setLineBegin(p+1);
52
f65a83c27638 - Fixed the raw string literal scanner. Newlines weren't copied to the buffer. Converting LS and PS to '\n' as well.
aziz
parents: 51
diff changeset
1383 break;
33
cf3047cf3cd2 - Added code for parsing back quote and raw strings.
aziz
parents: 32
diff changeset
1384 case '`':
cf3047cf3cd2 - Added code for parsing back quote and raw strings.
aziz
parents: 32
diff changeset
1385 case '"':
cf3047cf3cd2 - Added code for parsing back quote and raw strings.
aziz
parents: 32
diff changeset
1386 if (c == delim)
cf3047cf3cd2 - Added code for parsing back quote and raw strings.
aziz
parents: 32
diff changeset
1387 {
cf3047cf3cd2 - Added code for parsing back quote and raw strings.
aziz
parents: 32
diff changeset
1388 ++p;
cf3047cf3cd2 - Added code for parsing back quote and raw strings.
aziz
parents: 32
diff changeset
1389 t.pf = scanPostfix();
cf3047cf3cd2 - Added code for parsing back quote and raw strings.
aziz
parents: 32
diff changeset
1390 Lreturn:
cf3047cf3cd2 - Added code for parsing back quote and raw strings.
aziz
parents: 32
diff changeset
1391 t.str = buffer ~ '\0';
cf3047cf3cd2 - Added code for parsing back quote and raw strings.
aziz
parents: 32
diff changeset
1392 t.end = p;
cf3047cf3cd2 - Added code for parsing back quote and raw strings.
aziz
parents: 32
diff changeset
1393 return;
cf3047cf3cd2 - Added code for parsing back quote and raw strings.
aziz
parents: 32
diff changeset
1394 }
cf3047cf3cd2 - Added code for parsing back quote and raw strings.
aziz
parents: 32
diff changeset
1395 break;
cf3047cf3cd2 - Added code for parsing back quote and raw strings.
aziz
parents: 32
diff changeset
1396 case 0, _Z_:
cf3047cf3cd2 - Added code for parsing back quote and raw strings.
aziz
parents: 32
diff changeset
1397 if (delim == 'r')
422
ad7977fe315a Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 420
diff changeset
1398 error(tokenLineNum, tokenLineBegin, t.start, MID.UnterminatedRawString);
33
cf3047cf3cd2 - Added code for parsing back quote and raw strings.
aziz
parents: 32
diff changeset
1399 else
422
ad7977fe315a Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 420
diff changeset
1400 error(tokenLineNum, tokenLineBegin, t.start, MID.UnterminatedBackQuoteString);
33
cf3047cf3cd2 - Added code for parsing back quote and raw strings.
aziz
parents: 32
diff changeset
1401 goto Lreturn;
cf3047cf3cd2 - Added code for parsing back quote and raw strings.
aziz
parents: 32
diff changeset
1402 default:
383
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1403 if (c & 128)
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1404 {
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1405 c = decodeUTF8();
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1406 if (c == LSd || c == PSd)
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1407 goto case '\n';
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1408 encodeUTF8(buffer, c);
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1409 continue;
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1410 }
33
cf3047cf3cd2 - Added code for parsing back quote and raw strings.
aziz
parents: 32
diff changeset
1411 }
424
bb3cb00feeb2 Applied some fixes to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 422
diff changeset
1412 assert(isascii(c));
bb3cb00feeb2 Applied some fixes to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 422
diff changeset
1413 buffer ~= c;
33
cf3047cf3cd2 - Added code for parsing back quote and raw strings.
aziz
parents: 32
diff changeset
1414 }
cf3047cf3cd2 - Added code for parsing back quote and raw strings.
aziz
parents: 32
diff changeset
1415 assert(0);
31
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix.
aziz
parents: 30
diff changeset
1416 }
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix.
aziz
parents: 30
diff changeset
1417
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix.
aziz
parents: 30
diff changeset
1418 void scanHexStringLiteral(ref Token t)
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix.
aziz
parents: 30
diff changeset
1419 {
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix.
aziz
parents: 30
diff changeset
1420 assert(p[0] == 'x' && p[1] == '"');
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix.
aziz
parents: 30
diff changeset
1421 t.type = TOK.String;
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix.
aziz
parents: 30
diff changeset
1422
422
ad7977fe315a Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 420
diff changeset
1423 auto tokenLineNum = loc;
ad7977fe315a Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 420
diff changeset
1424 auto tokenLineBegin = lineBegin;
ad7977fe315a Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 420
diff changeset
1425
31
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix.
aziz
parents: 30
diff changeset
1426 uint c;
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix.
aziz
parents: 30
diff changeset
1427 ubyte[] buffer;
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix.
aziz
parents: 30
diff changeset
1428 ubyte h; // hex number
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix.
aziz
parents: 30
diff changeset
1429 uint n; // number of hex digits
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix.
aziz
parents: 30
diff changeset
1430
53
1786c2825491 - Fixed scanner for hex string literals. Terminating string with 0. Relocated some code to the bottom.
aziz
parents: 52
diff changeset
1431 ++p;
422
ad7977fe315a Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 420
diff changeset
1432 assert(*p == '"');
31
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix.
aziz
parents: 30
diff changeset
1433 while (1)
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix.
aziz
parents: 30
diff changeset
1434 {
53
1786c2825491 - Fixed scanner for hex string literals. Terminating string with 0. Relocated some code to the bottom.
aziz
parents: 52
diff changeset
1435 c = *++p;
31
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix.
aziz
parents: 30
diff changeset
1436 switch (c)
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix.
aziz
parents: 30
diff changeset
1437 {
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix.
aziz
parents: 30
diff changeset
1438 case '"':
53
1786c2825491 - Fixed scanner for hex string literals. Terminating string with 0. Relocated some code to the bottom.
aziz
parents: 52
diff changeset
1439 ++p;
31
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix.
aziz
parents: 30
diff changeset
1440 if (n & 1)
422
ad7977fe315a Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 420
diff changeset
1441 error(tokenLineNum, tokenLineBegin, t.start, MID.OddNumberOfDigitsInHexString);
53
1786c2825491 - Fixed scanner for hex string literals. Terminating string with 0. Relocated some code to the bottom.
aziz
parents: 52
diff changeset
1442 t.pf = scanPostfix();
1786c2825491 - Fixed scanner for hex string literals. Terminating string with 0. Relocated some code to the bottom.
aziz
parents: 52
diff changeset
1443 Lreturn:
1786c2825491 - Fixed scanner for hex string literals. Terminating string with 0. Relocated some code to the bottom.
aziz
parents: 52
diff changeset
1444 buffer ~= 0;
31
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix.
aziz
parents: 30
diff changeset
1445 t.str = cast(string) buffer;
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix.
aziz
parents: 30
diff changeset
1446 t.end = p;
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix.
aziz
parents: 30
diff changeset
1447 return;
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix.
aziz
parents: 30
diff changeset
1448 case '\r':
53
1786c2825491 - Fixed scanner for hex string literals. Terminating string with 0. Relocated some code to the bottom.
aziz
parents: 52
diff changeset
1449 if (p[1] == '\n')
31
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix.
aziz
parents: 30
diff changeset
1450 ++p;
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix.
aziz
parents: 30
diff changeset
1451 case '\n':
419
89e40d43065d Added new member 'p_newl' to Lexer and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 418
diff changeset
1452 assert(*p == '\n' || *p == '\r' || *p == LS[2] || *p == PS[2]);
31
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix.
aziz
parents: 30
diff changeset
1453 ++loc;
422
ad7977fe315a Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 420
diff changeset
1454 setLineBegin(p+1);
31
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix.
aziz
parents: 30
diff changeset
1455 continue;
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix.
aziz
parents: 30
diff changeset
1456 default:
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix.
aziz
parents: 30
diff changeset
1457 if (ishexad(c))
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix.
aziz
parents: 30
diff changeset
1458 {
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix.
aziz
parents: 30
diff changeset
1459 if (c <= '9')
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix.
aziz
parents: 30
diff changeset
1460 c -= '0';
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix.
aziz
parents: 30
diff changeset
1461 else if (c <= 'F')
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix.
aziz
parents: 30
diff changeset
1462 c -= 'A' - 10;
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix.
aziz
parents: 30
diff changeset
1463 else
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix.
aziz
parents: 30
diff changeset
1464 c -= 'a' - 10;
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix.
aziz
parents: 30
diff changeset
1465
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix.
aziz
parents: 30
diff changeset
1466 if (n & 1)
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix.
aziz
parents: 30
diff changeset
1467 {
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix.
aziz
parents: 30
diff changeset
1468 h <<= 4;
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix.
aziz
parents: 30
diff changeset
1469 h |= c;
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix.
aziz
parents: 30
diff changeset
1470 buffer ~= h;
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix.
aziz
parents: 30
diff changeset
1471 }
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix.
aziz
parents: 30
diff changeset
1472 else
112
004d98df65af - Implemented parseInterfaceDeclaration().
aziz
parents: 105
diff changeset
1473 h = cast(ubyte)c;
31
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix.
aziz
parents: 30
diff changeset
1474 ++n;
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix.
aziz
parents: 30
diff changeset
1475 continue;
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix.
aziz
parents: 30
diff changeset
1476 }
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix.
aziz
parents: 30
diff changeset
1477 else if (isspace(c))
419
89e40d43065d Added new member 'p_newl' to Lexer and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 418
diff changeset
1478 continue; // Skip spaces.
422
ad7977fe315a Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 420
diff changeset
1479 else if (c == LS[0] && p[1] == LS[1] && (p[2] == LS[2] || p[2] == PS[2]))
53
1786c2825491 - Fixed scanner for hex string literals. Terminating string with 0. Relocated some code to the bottom.
aziz
parents: 52
diff changeset
1480 {
422
ad7977fe315a Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 420
diff changeset
1481 ++p; ++p;
ad7977fe315a Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 420
diff changeset
1482 goto case '\n';
53
1786c2825491 - Fixed scanner for hex string literals. Terminating string with 0. Relocated some code to the bottom.
aziz
parents: 52
diff changeset
1483 }
1786c2825491 - Fixed scanner for hex string literals. Terminating string with 0. Relocated some code to the bottom.
aziz
parents: 52
diff changeset
1484 else if (c == 0 || c == _Z_)
1786c2825491 - Fixed scanner for hex string literals. Terminating string with 0. Relocated some code to the bottom.
aziz
parents: 52
diff changeset
1485 {
422
ad7977fe315a Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 420
diff changeset
1486 error(tokenLineNum, tokenLineBegin, t.start, MID.UnterminatedHexString);
53
1786c2825491 - Fixed scanner for hex string literals. Terminating string with 0. Relocated some code to the bottom.
aziz
parents: 52
diff changeset
1487 t.pf = 0;
1786c2825491 - Fixed scanner for hex string literals. Terminating string with 0. Relocated some code to the bottom.
aziz
parents: 52
diff changeset
1488 goto Lreturn;
1786c2825491 - Fixed scanner for hex string literals. Terminating string with 0. Relocated some code to the bottom.
aziz
parents: 52
diff changeset
1489 }
422
ad7977fe315a Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 420
diff changeset
1490 else
ad7977fe315a Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 420
diff changeset
1491 {
ad7977fe315a Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 420
diff changeset
1492 auto errorAt = p;
ad7977fe315a Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 420
diff changeset
1493 if (c & 128)
ad7977fe315a Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 420
diff changeset
1494 c = decodeUTF8();
ad7977fe315a Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 420
diff changeset
1495 error(errorAt, MID.NonHexCharInHexString, cast(dchar)c);
ad7977fe315a Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 420
diff changeset
1496 }
31
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix.
aziz
parents: 30
diff changeset
1497 }
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix.
aziz
parents: 30
diff changeset
1498 }
53
1786c2825491 - Fixed scanner for hex string literals. Terminating string with 0. Relocated some code to the bottom.
aziz
parents: 52
diff changeset
1499 assert(0);
31
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix.
aziz
parents: 30
diff changeset
1500 }
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix.
aziz
parents: 30
diff changeset
1501
383
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1502 version(D2)
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1503 {
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1504 void scanDelimitedStringLiteral(ref Token t)
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1505 {
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1506 assert(p[0] == 'q' && p[1] == '"');
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1507 t.type = TOK.String;
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1508
422
ad7977fe315a Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 420
diff changeset
1509 auto tokenLineNum = loc;
ad7977fe315a Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 420
diff changeset
1510 auto tokenLineBegin = lineBegin;
ad7977fe315a Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 420
diff changeset
1511
383
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1512 char[] buffer;
392
bb935c6f9b7a Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 391
diff changeset
1513 dchar opening_delim = 0, // 0 if no nested delimiter or '[', '(', '<', '{'
bb935c6f9b7a Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 391
diff changeset
1514 closing_delim; // Will be ']', ')', '>', '},
bb935c6f9b7a Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 391
diff changeset
1515 // the first character of an identifier or
bb935c6f9b7a Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 391
diff changeset
1516 // any other Unicode/ASCII character.
bb935c6f9b7a Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 391
diff changeset
1517 char[] str_delim; // Identifier delimiter.
bb935c6f9b7a Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 391
diff changeset
1518 uint level = 1; // Counter for nestable delimiters.
383
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1519
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1520 ++p; ++p; // Skip q"
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1521 uint c = *p;
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1522 switch (c)
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1523 {
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1524 case '(':
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1525 opening_delim = c;
392
bb935c6f9b7a Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 391
diff changeset
1526 closing_delim = ')'; // c + 1
383
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1527 break;
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1528 case '[', '<', '{':
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1529 opening_delim = c;
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1530 closing_delim = c + 2; // Get to closing counterpart. Feature of ASCII table.
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1531 break;
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1532 default:
392
bb935c6f9b7a Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 391
diff changeset
1533 dchar scanNewline()
bb935c6f9b7a Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 391
diff changeset
1534 {
bb935c6f9b7a Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 391
diff changeset
1535 switch (*p)
bb935c6f9b7a Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 391
diff changeset
1536 {
bb935c6f9b7a Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 391
diff changeset
1537 case '\r':
bb935c6f9b7a Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 391
diff changeset
1538 if (p[1] == '\n')
bb935c6f9b7a Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 391
diff changeset
1539 ++p;
bb935c6f9b7a Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 391
diff changeset
1540 case '\n':
419
89e40d43065d Added new member 'p_newl' to Lexer and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 418
diff changeset
1541 assert(*p == '\n' || *p == '\r' || *p == LS[2] || *p == PS[2]);
392
bb935c6f9b7a Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 391
diff changeset
1542 ++p;
bb935c6f9b7a Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 391
diff changeset
1543 ++loc;
422
ad7977fe315a Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 420
diff changeset
1544 setLineBegin(p);
392
bb935c6f9b7a Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 391
diff changeset
1545 return '\n';
bb935c6f9b7a Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 391
diff changeset
1546 case LS[0]:
bb935c6f9b7a Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 391
diff changeset
1547 if (p[1] == LS[1] && (p[2] == LS[2] || p[2] == PS[2]))
bb935c6f9b7a Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 391
diff changeset
1548 {
419
89e40d43065d Added new member 'p_newl' to Lexer and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 418
diff changeset
1549 ++p; ++p;
89e40d43065d Added new member 'p_newl' to Lexer and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 418
diff changeset
1550 goto case '\n';
392
bb935c6f9b7a Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 391
diff changeset
1551 }
bb935c6f9b7a Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 391
diff changeset
1552 default:
bb935c6f9b7a Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 391
diff changeset
1553 }
bb935c6f9b7a Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 391
diff changeset
1554 return 0;
bb935c6f9b7a Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 391
diff changeset
1555 }
bb935c6f9b7a Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 391
diff changeset
1556
bb935c6f9b7a Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 391
diff changeset
1557 // Skip leading newlines:
bb935c6f9b7a Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 391
diff changeset
1558 while (scanNewline() != 0){}
419
89e40d43065d Added new member 'p_newl' to Lexer and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 418
diff changeset
1559 assert(*p != '\n' && *p != '\r' &&
89e40d43065d Added new member 'p_newl' to Lexer and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 418
diff changeset
1560 !(*p == LS[0] && p[1] == LS[1] && (p[2] == LS[2] || p[2] == PS[2])));
392
bb935c6f9b7a Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 391
diff changeset
1561
383
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1562 char* begin = p;
392
bb935c6f9b7a Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 391
diff changeset
1563 c = *p;
383
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1564 closing_delim = c;
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1565 // TODO: Check for non-printable characters?
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1566 if (c & 128)
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1567 {
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1568 closing_delim = decodeUTF8();
392
bb935c6f9b7a Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 391
diff changeset
1569 if (!isUniAlpha(closing_delim))
bb935c6f9b7a Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 391
diff changeset
1570 break; // Not an identifier.
383
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1571 }
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1572 else if (!isidbeg(c))
392
bb935c6f9b7a Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 391
diff changeset
1573 break; // Not an identifier.
bb935c6f9b7a Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 391
diff changeset
1574
bb935c6f9b7a Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 391
diff changeset
1575 // Parse Identifier + EndOfLine
383
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1576 do
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1577 { c = *++p; }
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1578 while (isident(c) || c & 128 && isUniAlpha(decodeUTF8()))
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1579 // Store identifier
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1580 str_delim = begin[0..p-begin];
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1581 // Scan newline
392
bb935c6f9b7a Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 391
diff changeset
1582 if (scanNewline() == '\n')
bb935c6f9b7a Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 391
diff changeset
1583 --p; // Go back one because of "c = *++p;" in main loop.
bb935c6f9b7a Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 391
diff changeset
1584 else
383
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1585 {
422
ad7977fe315a Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 420
diff changeset
1586 // TODO: error(p, MID.ExpectedNewlineAfterIdentDelim);
383
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1587 }
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1588 }
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1589
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1590 bool checkStringDelim(char* p)
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1591 {
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1592 assert(str_delim.length != 0);
392
bb935c6f9b7a Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 391
diff changeset
1593 if (buffer[$-1] == '\n' && // Last character copied to buffer must be '\n'.
bb935c6f9b7a Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 391
diff changeset
1594 end-p >= str_delim.length && // Check remaining length.
383
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1595 p[0..str_delim.length] == str_delim) // Compare.
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1596 return true;
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1597 return false;
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1598 }
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1599
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1600 while (1)
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1601 {
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1602 c = *++p;
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1603 switch (c)
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1604 {
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1605 case '\r':
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1606 if (p[1] == '\n')
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1607 ++p;
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1608 case '\n':
419
89e40d43065d Added new member 'p_newl' to Lexer and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 418
diff changeset
1609 assert(*p == '\n' || *p == '\r' || *p == LS[2] || *p == PS[2]);
383
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1610 c = '\n'; // Convert EndOfLine ('\r','\r\n','\n',LS,PS) to '\n'
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1611 ++loc;
422
ad7977fe315a Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 420
diff changeset
1612 setLineBegin(p+1);
383
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1613 break;
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1614 case 0, _Z_:
422
ad7977fe315a Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 420
diff changeset
1615 // TODO: error(tokenLineNum, tokenLineBegin, t.start, MID.UnterminatedDelimitedString);
383
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1616 goto Lreturn3;
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1617 default:
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1618 if (c & 128)
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1619 {
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1620 auto begin = p;
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1621 c = decodeUTF8();
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1622 if (c == LSd || c == PSd)
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1623 goto case '\n';
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1624 if (c == closing_delim)
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1625 {
392
bb935c6f9b7a Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 391
diff changeset
1626 if (str_delim.length)
383
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1627 {
392
bb935c6f9b7a Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 391
diff changeset
1628 if (checkStringDelim(begin))
bb935c6f9b7a Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 391
diff changeset
1629 {
bb935c6f9b7a Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 391
diff changeset
1630 p = begin + str_delim.length;
bb935c6f9b7a Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 391
diff changeset
1631 goto Lreturn2;
bb935c6f9b7a Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 391
diff changeset
1632 }
383
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1633 }
392
bb935c6f9b7a Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 391
diff changeset
1634 else
bb935c6f9b7a Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 391
diff changeset
1635 {
bb935c6f9b7a Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 391
diff changeset
1636 assert(level == 1);
bb935c6f9b7a Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 391
diff changeset
1637 --level;
bb935c6f9b7a Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 391
diff changeset
1638 goto Lreturn;
bb935c6f9b7a Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 391
diff changeset
1639 }
383
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1640 }
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1641 encodeUTF8(buffer, c);
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1642 continue;
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1643 }
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1644 else
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1645 {
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1646 if (c == opening_delim)
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1647 ++level;
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1648 else if (c == closing_delim)
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1649 {
392
bb935c6f9b7a Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 391
diff changeset
1650 if (str_delim.length)
383
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1651 {
392
bb935c6f9b7a Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 391
diff changeset
1652 if (checkStringDelim(p))
bb935c6f9b7a Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 391
diff changeset
1653 {
bb935c6f9b7a Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 391
diff changeset
1654 p += str_delim.length;
bb935c6f9b7a Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 391
diff changeset
1655 goto Lreturn2;
bb935c6f9b7a Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 391
diff changeset
1656 }
383
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1657 }
392
bb935c6f9b7a Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 391
diff changeset
1658 else if (--level == 0)
383
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1659 goto Lreturn;
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1660 }
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1661 }
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1662 }
424
bb3cb00feeb2 Applied some fixes to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 422
diff changeset
1663 assert(isascii(c));
bb3cb00feeb2 Applied some fixes to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 422
diff changeset
1664 buffer ~= c;
383
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1665 }
392
bb935c6f9b7a Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 391
diff changeset
1666 Lreturn: // Character delimiter.
bb935c6f9b7a Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 391
diff changeset
1667 assert(c == closing_delim);
383
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1668 assert(level == 0);
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1669 ++p; // Skip closing delimiter.
392
bb935c6f9b7a Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 391
diff changeset
1670 Lreturn2: // String delimiter.
383
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1671 if (*p == '"')
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1672 ++p;
422
ad7977fe315a Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 420
diff changeset
1673 else
ad7977fe315a Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 420
diff changeset
1674 {
ad7977fe315a Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 420
diff changeset
1675 // TODO: error(p, MID.ExpectedDblQuoteAfterDelim, str_delim.length ? str_delim : closing_delim~"");
ad7977fe315a Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 420
diff changeset
1676 }
383
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1677
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1678 t.pf = scanPostfix();
392
bb935c6f9b7a Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 391
diff changeset
1679 Lreturn3: // Error.
383
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1680 t.str = buffer ~ '\0';
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1681 t.end = p;
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1682 }
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1683
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1684 void scanTokenStringLiteral(ref Token t)
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1685 {
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1686 assert(p[0] == 'q' && p[1] == '{');
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1687 t.type = TOK.String;
392
bb935c6f9b7a Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 391
diff changeset
1688
422
ad7977fe315a Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 420
diff changeset
1689 auto tokenLineNum = loc;
ad7977fe315a Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 420
diff changeset
1690 auto tokenLineBegin = lineBegin;
ad7977fe315a Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 420
diff changeset
1691
392
bb935c6f9b7a Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 391
diff changeset
1692 // A guard against changes to particular members:
422
ad7977fe315a Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 420
diff changeset
1693 // this.loc_old, this.loc_hline and this.errorLoc.filePath
392
bb935c6f9b7a Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 391
diff changeset
1694 ++inTokenString;
383
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1695
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1696 uint loc = this.loc;
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1697 uint level = 1;
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1698
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1699 ++p; ++p; // Skip q{
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1700
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1701 auto prev_t = &t;
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1702 Token* token;
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1703 while (1)
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1704 {
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1705 token = new Token;
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1706 scan(*token);
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1707 // Save the tokens in a doubly linked list.
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1708 // Could be useful for various tools.
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1709 token.prev = prev_t;
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1710 prev_t.next = token;
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1711 prev_t = token;
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1712 switch (token.type)
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1713 {
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1714 case TOK.LBrace:
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1715 ++level;
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1716 continue;
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1717 case TOK.RBrace:
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1718 if (--level == 0)
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1719 {
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1720 t.tok_str = t.next;
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1721 t.next = null;
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1722 break;
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1723 }
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1724 continue;
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1725 case TOK.EOF:
422
ad7977fe315a Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 420
diff changeset
1726 // TODO: error(tokenLineNum, tokenLineBegin, t.start, MID.UnterminatedTokenString);
383
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1727 t.tok_str = t.next;
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1728 t.next = token;
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1729 break;
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1730 default:
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1731 continue;
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1732 }
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1733 break; // Exit loop.
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1734 }
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1735
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1736 assert(token.type == TOK.RBrace || token.type == TOK.EOF);
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1737 assert(token.type == TOK.RBrace && t.next is null ||
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1738 token.type == TOK.EOF && t.next !is null);
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1739
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1740 char[] buffer;
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1741 // token points to } or EOF
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1742 if (token.type == TOK.EOF)
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1743 {
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1744 t.end = token.start;
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1745 buffer = t.srcText[2..$].dup ~ '\0';
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1746 }
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1747 else
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1748 {
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1749 // Assign to buffer before scanPostfix().
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1750 t.end = p;
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1751 buffer = t.srcText[2..$-1].dup ~ '\0';
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1752 t.pf = scanPostfix();
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1753 t.end = p;
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1754 }
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1755 // Convert EndOfLines to '\n'
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1756 if (loc != this.loc)
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1757 {
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1758 assert(buffer[$-1] == '\0');
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1759 uint i, j;
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1760 for (; i < buffer.length; ++i)
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1761 switch (buffer[i])
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1762 {
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1763 case '\r':
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1764 if (buffer[i+1] == '\n')
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1765 ++i;
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1766 case '\n':
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1767 buffer[j++] = '\n';
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1768 break;
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1769 case LS[0]:
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1770 auto b = buffer[i..$];
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1771 if (b[1] == LS[1] && (b[2] == LS[2] || b[2] == PS[2]))
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1772 {
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1773 ++i; ++i;
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1774 goto case '\n';
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1775 }
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1776 // goto default;
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1777 default:
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1778 buffer[j++] = buffer[i]; // Copy character
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1779 }
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1780 buffer.length = j; // Adjust length
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1781 }
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1782 assert(buffer[$-1] == '\0');
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1783 t.str = buffer;
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1784
392
bb935c6f9b7a Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 391
diff changeset
1785 --inTokenString;
383
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1786 }
392
bb935c6f9b7a Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 391
diff changeset
1787 } // version(D2)
383
6a5fc22cae34 Implemented scanner for new string literals and applied some fixes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 382
diff changeset
1788
45
cc84b9ca9d0a - Implemented escape sequence scanner.
aziz
parents: 44
diff changeset
1789 dchar scanEscapeSequence()
cc84b9ca9d0a - Implemented escape sequence scanner.
aziz
parents: 44
diff changeset
1790 {
386
392a0068fc61 Refactored code related to scanning escape sequences.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 384
diff changeset
1791 assert(*p == '\\');
422
ad7977fe315a Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 420
diff changeset
1792
ad7977fe315a Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 420
diff changeset
1793 auto sequenceStart = p; // Used for error reporting.
ad7977fe315a Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 420
diff changeset
1794
386
392a0068fc61 Refactored code related to scanning escape sequences.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 384
diff changeset
1795 ++p;
45
cc84b9ca9d0a - Implemented escape sequence scanner.
aziz
parents: 44
diff changeset
1796 uint c = char2ev(*p);
82
fc645fb2fe72 - scanEscapeSequences() doesn't return 0xFFFF as an error value anymore, because it is a valid codepoint usable by the user.
aziz
parents: 71
diff changeset
1797 if (c)
fc645fb2fe72 - scanEscapeSequences() doesn't return 0xFFFF as an error value anymore, because it is a valid codepoint usable by the user.
aziz
parents: 71
diff changeset
1798 {
46
e0dbd4722f74 - Scanning character literals correctly now.
aziz
parents: 45
diff changeset
1799 ++p;
45
cc84b9ca9d0a - Implemented escape sequence scanner.
aziz
parents: 44
diff changeset
1800 return c;
46
e0dbd4722f74 - Scanning character literals correctly now.
aziz
parents: 45
diff changeset
1801 }
45
cc84b9ca9d0a - Implemented escape sequence scanner.
aziz
parents: 44
diff changeset
1802 uint digits = 2;
cc84b9ca9d0a - Implemented escape sequence scanner.
aziz
parents: 44
diff changeset
1803
cc84b9ca9d0a - Implemented escape sequence scanner.
aziz
parents: 44
diff changeset
1804 switch (*p)
cc84b9ca9d0a - Implemented escape sequence scanner.
aziz
parents: 44
diff changeset
1805 {
cc84b9ca9d0a - Implemented escape sequence scanner.
aziz
parents: 44
diff changeset
1806 case 'x':
cc84b9ca9d0a - Implemented escape sequence scanner.
aziz
parents: 44
diff changeset
1807 c = 0;
cc84b9ca9d0a - Implemented escape sequence scanner.
aziz
parents: 44
diff changeset
1808 while (1)
cc84b9ca9d0a - Implemented escape sequence scanner.
aziz
parents: 44
diff changeset
1809 {
cc84b9ca9d0a - Implemented escape sequence scanner.
aziz
parents: 44
diff changeset
1810 ++p;
cc84b9ca9d0a - Implemented escape sequence scanner.
aziz
parents: 44
diff changeset
1811 if (ishexad(*p))
cc84b9ca9d0a - Implemented escape sequence scanner.
aziz
parents: 44
diff changeset
1812 {
cc84b9ca9d0a - Implemented escape sequence scanner.
aziz
parents: 44
diff changeset
1813 c *= 16;
cc84b9ca9d0a - Implemented escape sequence scanner.
aziz
parents: 44
diff changeset
1814 if (*p <= '9')
48
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
1815 c += *p - '0';
45
cc84b9ca9d0a - Implemented escape sequence scanner.
aziz
parents: 44
diff changeset
1816 else if (*p <= 'F')
48
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
1817 c += *p - 'A' + 10;
45
cc84b9ca9d0a - Implemented escape sequence scanner.
aziz
parents: 44
diff changeset
1818 else
48
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
1819 c += *p - 'a' + 10;
82
fc645fb2fe72 - scanEscapeSequences() doesn't return 0xFFFF as an error value anymore, because it is a valid codepoint usable by the user.
aziz
parents: 71
diff changeset
1820
fc645fb2fe72 - scanEscapeSequences() doesn't return 0xFFFF as an error value anymore, because it is a valid codepoint usable by the user.
aziz
parents: 71
diff changeset
1821 if (!--digits)
fc645fb2fe72 - scanEscapeSequences() doesn't return 0xFFFF as an error value anymore, because it is a valid codepoint usable by the user.
aziz
parents: 71
diff changeset
1822 {
48
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
1823 ++p;
45
cc84b9ca9d0a - Implemented escape sequence scanner.
aziz
parents: 44
diff changeset
1824 break;
48
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
1825 }
45
cc84b9ca9d0a - Implemented escape sequence scanner.
aziz
parents: 44
diff changeset
1826 }
cc84b9ca9d0a - Implemented escape sequence scanner.
aziz
parents: 44
diff changeset
1827 else
cc84b9ca9d0a - Implemented escape sequence scanner.
aziz
parents: 44
diff changeset
1828 {
422
ad7977fe315a Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 420
diff changeset
1829 error(sequenceStart, MID.InsufficientHexDigits);
432
3ead178e0662 Added message MID.InvalidUnicodeEscapeSequence.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 430
diff changeset
1830 return REPLACEMENT_CHAR;
45
cc84b9ca9d0a - Implemented escape sequence scanner.
aziz
parents: 44
diff changeset
1831 }
cc84b9ca9d0a - Implemented escape sequence scanner.
aziz
parents: 44
diff changeset
1832 }
432
3ead178e0662 Added message MID.InvalidUnicodeEscapeSequence.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 430
diff changeset
1833
424
bb3cb00feeb2 Applied some fixes to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 422
diff changeset
1834 if (!isEncodable(c))
bb3cb00feeb2 Applied some fixes to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 422
diff changeset
1835 {
430
e6c759e151cd Fixed a few things regarding encoding/decoding UTF-8 sequences.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 426
diff changeset
1836 c = REPLACEMENT_CHAR;
432
3ead178e0662 Added message MID.InvalidUnicodeEscapeSequence.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 430
diff changeset
1837 assert(*sequenceStart == '\\');
3ead178e0662 Added message MID.InvalidUnicodeEscapeSequence.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 430
diff changeset
1838 error(sequenceStart, MID.InvalidUnicodeEscapeSequence, sequenceStart[0..p-sequenceStart]);
424
bb3cb00feeb2 Applied some fixes to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 422
diff changeset
1839 }
422
ad7977fe315a Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 420
diff changeset
1840 return c;
45
cc84b9ca9d0a - Implemented escape sequence scanner.
aziz
parents: 44
diff changeset
1841 case 'u':
cc84b9ca9d0a - Implemented escape sequence scanner.
aziz
parents: 44
diff changeset
1842 digits = 4;
cc84b9ca9d0a - Implemented escape sequence scanner.
aziz
parents: 44
diff changeset
1843 goto case 'x';
cc84b9ca9d0a - Implemented escape sequence scanner.
aziz
parents: 44
diff changeset
1844 case 'U':
cc84b9ca9d0a - Implemented escape sequence scanner.
aziz
parents: 44
diff changeset
1845 digits = 8;
cc84b9ca9d0a - Implemented escape sequence scanner.
aziz
parents: 44
diff changeset
1846 goto case 'x';
cc84b9ca9d0a - Implemented escape sequence scanner.
aziz
parents: 44
diff changeset
1847 default:
48
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
1848 if (isoctal(*p))
45
cc84b9ca9d0a - Implemented escape sequence scanner.
aziz
parents: 44
diff changeset
1849 {
48
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
1850 c = 0;
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
1851 c += *p - '0';
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
1852 ++p;
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
1853 if (!isoctal(*p))
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
1854 return c;
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
1855 c *= 8;
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
1856 c += *p - '0';
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
1857 ++p;
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
1858 if (!isoctal(*p))
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
1859 return c;
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
1860 c *= 8;
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
1861 c += *p - '0';
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
1862 ++p;
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
1863 }
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
1864 else if(*p == '&')
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
1865 {
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
1866 if (isalpha(*++p))
45
cc84b9ca9d0a - Implemented escape sequence scanner.
aziz
parents: 44
diff changeset
1867 {
272
0bde32503976 - Added module HtmlEntities. It contains a table for converting HTML entities to Unicode characters.
aziz
parents: 249
diff changeset
1868 auto begin = p;
0bde32503976 - Added module HtmlEntities. It contains a table for converting HTML entities to Unicode characters.
aziz
parents: 249
diff changeset
1869 while (isalnum(*++p))
0bde32503976 - Added module HtmlEntities. It contains a table for converting HTML entities to Unicode characters.
aziz
parents: 249
diff changeset
1870 {}
0bde32503976 - Added module HtmlEntities. It contains a table for converting HTML entities to Unicode characters.
aziz
parents: 249
diff changeset
1871
0bde32503976 - Added module HtmlEntities. It contains a table for converting HTML entities to Unicode characters.
aziz
parents: 249
diff changeset
1872 if (*p == ';')
48
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
1873 {
422
ad7977fe315a Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 420
diff changeset
1874 // Pass entity excluding '&' and ';'.
272
0bde32503976 - Added module HtmlEntities. It contains a table for converting HTML entities to Unicode characters.
aziz
parents: 249
diff changeset
1875 c = entity2Unicode(begin[0..p - begin]);
386
392a0068fc61 Refactored code related to scanning escape sequences.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 384
diff changeset
1876 ++p; // Skip ;
272
0bde32503976 - Added module HtmlEntities. It contains a table for converting HTML entities to Unicode characters.
aziz
parents: 249
diff changeset
1877 if (c == 0xFFFF)
422
ad7977fe315a Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 420
diff changeset
1878 error(sequenceStart, MID.UndefinedHTMLEntity, sequenceStart[0 .. p - sequenceStart]);
272
0bde32503976 - Added module HtmlEntities. It contains a table for converting HTML entities to Unicode characters.
aziz
parents: 249
diff changeset
1879 }
0bde32503976 - Added module HtmlEntities. It contains a table for converting HTML entities to Unicode characters.
aziz
parents: 249
diff changeset
1880 else
432
3ead178e0662 Added message MID.InvalidUnicodeEscapeSequence.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 430
diff changeset
1881 error(sequenceStart, MID.UnterminatedHTMLEntity, sequenceStart[0 .. p - sequenceStart]);
45
cc84b9ca9d0a - Implemented escape sequence scanner.
aziz
parents: 44
diff changeset
1882 }
48
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
1883 else
422
ad7977fe315a Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 420
diff changeset
1884 error(sequenceStart, MID.InvalidBeginHTMLEntity);
45
cc84b9ca9d0a - Implemented escape sequence scanner.
aziz
parents: 44
diff changeset
1885 }
416
a4783b904fba Detecting two more errors when scanning an escape sequence.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 414
diff changeset
1886 else if (*p == '\n' || *p == '\r' ||
a4783b904fba Detecting two more errors when scanning an escape sequence.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 414
diff changeset
1887 *p == LS[0] && p[1] == LS[1] && (p[2] == LS[2] || p[2] == PS[2]))
a4783b904fba Detecting two more errors when scanning an escape sequence.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 414
diff changeset
1888 {
422
ad7977fe315a Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 420
diff changeset
1889 error(sequenceStart, MID.UndefinedEscapeSequence, r"\NewLine");
416
a4783b904fba Detecting two more errors when scanning an escape sequence.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 414
diff changeset
1890 }
a4783b904fba Detecting two more errors when scanning an escape sequence.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 414
diff changeset
1891 else if (*p == 0 || *p == _Z_)
a4783b904fba Detecting two more errors when scanning an escape sequence.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 414
diff changeset
1892 {
422
ad7977fe315a Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 420
diff changeset
1893 error(sequenceStart, MID.UndefinedEscapeSequence, r"\EOF");
416
a4783b904fba Detecting two more errors when scanning an escape sequence.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 414
diff changeset
1894 }
45
cc84b9ca9d0a - Implemented escape sequence scanner.
aziz
parents: 44
diff changeset
1895 else
386
392a0068fc61 Refactored code related to scanning escape sequences.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 384
diff changeset
1896 {
392a0068fc61 Refactored code related to scanning escape sequences.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 384
diff changeset
1897 char[] str = `\`;
424
bb3cb00feeb2 Applied some fixes to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 422
diff changeset
1898 if (*p & 128)
387
ad0cbd1c8881 Undefined escape sequences are passed to error() now.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 386
diff changeset
1899 encodeUTF8(str, decodeUTF8());
386
392a0068fc61 Refactored code related to scanning escape sequences.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 384
diff changeset
1900 else
424
bb3cb00feeb2 Applied some fixes to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 422
diff changeset
1901 str ~= *p;
387
ad0cbd1c8881 Undefined escape sequences are passed to error() now.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 386
diff changeset
1902 ++p;
ad0cbd1c8881 Undefined escape sequences are passed to error() now.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 386
diff changeset
1903 // TODO: check for unprintable character?
422
ad7977fe315a Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 420
diff changeset
1904 error(sequenceStart, MID.UndefinedEscapeSequence, str);
386
392a0068fc61 Refactored code related to scanning escape sequences.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 384
diff changeset
1905 }
45
cc84b9ca9d0a - Implemented escape sequence scanner.
aziz
parents: 44
diff changeset
1906 }
cc84b9ca9d0a - Implemented escape sequence scanner.
aziz
parents: 44
diff changeset
1907
cc84b9ca9d0a - Implemented escape sequence scanner.
aziz
parents: 44
diff changeset
1908 return c;
cc84b9ca9d0a - Implemented escape sequence scanner.
aziz
parents: 44
diff changeset
1909 }
cc84b9ca9d0a - Implemented escape sequence scanner.
aziz
parents: 44
diff changeset
1910
56
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1911 /*
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1912 IntegerLiteral:= (Dec|Hex|Bin|Oct)Suffix?
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1913 Dec:= (0|[1-9][0-9_]*)
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1914 Hex:= 0[xX] HexDigits
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1915 Bin:= 0[bB][01_]+
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1916 Oct:= 0[0-7_]+
68
7eb83dd38901 - Simplified suffix rule and added a few more numbers to unittest.
aziz
parents: 67
diff changeset
1917 Suffix:= (L[uU]?|[uU]L?)
56
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1918 HexDigits:= [0-9a-zA-Z_]+
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1919
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1920 Invalid: "0b_", "0x_", "._"
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1921 */
15
c70c028e47dd - Started implementation of lexing numbers.
aziz
parents: 14
diff changeset
1922 void scanNumber(ref Token t)
c70c028e47dd - Started implementation of lexing numbers.
aziz
parents: 14
diff changeset
1923 {
56
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1924 ulong ulong_;
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1925 bool overflow;
58
50bb7fc9db44 - The types of integers are recognized now.
aziz
parents: 57
diff changeset
1926 bool isDecimal;
57
c0f1c8be3a47 - Added code for converting hex characters to binary numbers.
aziz
parents: 56
diff changeset
1927 size_t digits;
56
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1928
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1929 if (*p != '0')
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1930 goto LscanInteger;
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1931 ++p; // skip zero
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1932 // check for xX bB ...
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1933 switch (*p)
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1934 {
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1935 case 'x','X':
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1936 goto LscanHex;
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1937 case 'b','B':
396
0a4619735ce9 Applied fixes to Lexer, Parser and other classes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 393
diff changeset
1938 goto LscanBinary;
56
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1939 case 'L':
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1940 if (p[1] == 'i')
396
0a4619735ce9 Applied fixes to Lexer, Parser and other classes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 393
diff changeset
1941 goto LscanReal; // 0Li
0a4619735ce9 Applied fixes to Lexer, Parser and other classes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 393
diff changeset
1942 break; // 0L
56
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1943 case '.':
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1944 if (p[1] == '.')
396
0a4619735ce9 Applied fixes to Lexer, Parser and other classes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 393
diff changeset
1945 break; // 0..
0a4619735ce9 Applied fixes to Lexer, Parser and other classes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 393
diff changeset
1946 // 0.
392
bb935c6f9b7a Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 391
diff changeset
1947 case 'i','f','F', // Imaginary and float literal suffixes.
bb935c6f9b7a Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 391
diff changeset
1948 'e', 'E': // Float exponent.
56
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1949 goto LscanReal;
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1950 default:
396
0a4619735ce9 Applied fixes to Lexer, Parser and other classes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 393
diff changeset
1951 if (*p == '_')
0a4619735ce9 Applied fixes to Lexer, Parser and other classes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 393
diff changeset
1952 goto LscanOctal; // 0_
0a4619735ce9 Applied fixes to Lexer, Parser and other classes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 393
diff changeset
1953 else if (isdigit(*p))
0a4619735ce9 Applied fixes to Lexer, Parser and other classes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 393
diff changeset
1954 {
0a4619735ce9 Applied fixes to Lexer, Parser and other classes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 393
diff changeset
1955 if (*p == '8' || *p == '9')
0a4619735ce9 Applied fixes to Lexer, Parser and other classes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 393
diff changeset
1956 goto Loctal_hasDecimalDigits; // 08 or 09
0a4619735ce9 Applied fixes to Lexer, Parser and other classes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 393
diff changeset
1957 else
0a4619735ce9 Applied fixes to Lexer, Parser and other classes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 393
diff changeset
1958 goto Loctal_enter_loop; // 0[0-7]
0a4619735ce9 Applied fixes to Lexer, Parser and other classes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 393
diff changeset
1959 }
56
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1960 }
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1961
323
6259fb93e3dd - Rewrote scanSpecialToken().
aziz
parents: 315
diff changeset
1962 // Number 0
6259fb93e3dd - Rewrote scanSpecialToken().
aziz
parents: 315
diff changeset
1963 assert(p[-1] == '0');
396
0a4619735ce9 Applied fixes to Lexer, Parser and other classes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 393
diff changeset
1964 assert(*p != '_' && !isdigit(*p));
323
6259fb93e3dd - Rewrote scanSpecialToken().
aziz
parents: 315
diff changeset
1965 assert(ulong_ == 0);
58
50bb7fc9db44 - The types of integers are recognized now.
aziz
parents: 57
diff changeset
1966 isDecimal = true;
50bb7fc9db44 - The types of integers are recognized now.
aziz
parents: 57
diff changeset
1967 goto Lfinalize;
50bb7fc9db44 - The types of integers are recognized now.
aziz
parents: 57
diff changeset
1968
56
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1969 LscanInteger:
323
6259fb93e3dd - Rewrote scanSpecialToken().
aziz
parents: 315
diff changeset
1970 assert(*p != 0 && isdigit(*p));
58
50bb7fc9db44 - The types of integers are recognized now.
aziz
parents: 57
diff changeset
1971 isDecimal = true;
323
6259fb93e3dd - Rewrote scanSpecialToken().
aziz
parents: 315
diff changeset
1972 goto Lenter_loop_int;
63
c29229fbf2f7 - Recognizing floats that start with a dot.
aziz
parents: 62
diff changeset
1973 while (1)
56
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1974 {
63
c29229fbf2f7 - Recognizing floats that start with a dot.
aziz
parents: 62
diff changeset
1975 if (*++p == '_')
56
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1976 continue;
63
c29229fbf2f7 - Recognizing floats that start with a dot.
aziz
parents: 62
diff changeset
1977 if (!isdigit(*p))
c29229fbf2f7 - Recognizing floats that start with a dot.
aziz
parents: 62
diff changeset
1978 break;
323
6259fb93e3dd - Rewrote scanSpecialToken().
aziz
parents: 315
diff changeset
1979 Lenter_loop_int:
56
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1980 if (ulong_ < ulong.max/10 || (ulong_ == ulong.max/10 && *p <= '5'))
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1981 {
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1982 ulong_ *= 10;
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1983 ulong_ += *p - '0';
58
50bb7fc9db44 - The types of integers are recognized now.
aziz
parents: 57
diff changeset
1984 continue;
56
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1985 }
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1986 // Overflow: skip following digits.
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1987 overflow = true;
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1988 while (isdigit(*++p)) {}
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1989 break;
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1990 }
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1991
61
512cd2248dfc - Fix: issueing error on hexadecimal number overflow.
aziz
parents: 60
diff changeset
1992 // The number could be a float, so check overflow below.
56
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1993 switch (*p)
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1994 {
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1995 case '.':
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1996 if (p[1] != '.')
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1997 goto LscanReal;
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1998 break;
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
1999 case 'L':
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
2000 if (p[1] != 'i')
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
2001 break;
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
2002 case 'i', 'f', 'F', 'e', 'E':
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
2003 goto LscanReal;
58
50bb7fc9db44 - The types of integers are recognized now.
aziz
parents: 57
diff changeset
2004 default:
56
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
2005 }
58
50bb7fc9db44 - The types of integers are recognized now.
aziz
parents: 57
diff changeset
2006
56
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
2007 if (overflow)
422
ad7977fe315a Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 420
diff changeset
2008 error(t.start, MID.OverflowDecimalNumber);
58
50bb7fc9db44 - The types of integers are recognized now.
aziz
parents: 57
diff changeset
2009
56
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
2010 assert((isdigit(p[-1]) || p[-1] == '_') && !isdigit(*p) && *p != '_');
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
2011 goto Lfinalize;
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
2012
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
2013 LscanHex:
61
512cd2248dfc - Fix: issueing error on hexadecimal number overflow.
aziz
parents: 60
diff changeset
2014 assert(digits == 0);
410
4d9ee8e60712 Added destructors for two particular tokens.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 403
diff changeset
2015 assert(*p == 'x' || *p == 'X');
62
96af5653acef - Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents: 61
diff changeset
2016 while (1)
56
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
2017 {
62
96af5653acef - Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents: 61
diff changeset
2018 if (*++p == '_')
56
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
2019 continue;
62
96af5653acef - Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents: 61
diff changeset
2020 if (!ishexad(*p))
96af5653acef - Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents: 61
diff changeset
2021 break;
61
512cd2248dfc - Fix: issueing error on hexadecimal number overflow.
aziz
parents: 60
diff changeset
2022 ++digits;
57
c0f1c8be3a47 - Added code for converting hex characters to binary numbers.
aziz
parents: 56
diff changeset
2023 ulong_ *= 16;
c0f1c8be3a47 - Added code for converting hex characters to binary numbers.
aziz
parents: 56
diff changeset
2024 if (*p <= '9')
c0f1c8be3a47 - Added code for converting hex characters to binary numbers.
aziz
parents: 56
diff changeset
2025 ulong_ += *p - '0';
c0f1c8be3a47 - Added code for converting hex characters to binary numbers.
aziz
parents: 56
diff changeset
2026 else if (*p <= 'F')
c0f1c8be3a47 - Added code for converting hex characters to binary numbers.
aziz
parents: 56
diff changeset
2027 ulong_ += *p - 'A' + 10;
c0f1c8be3a47 - Added code for converting hex characters to binary numbers.
aziz
parents: 56
diff changeset
2028 else
c0f1c8be3a47 - Added code for converting hex characters to binary numbers.
aziz
parents: 56
diff changeset
2029 ulong_ += *p - 'a' + 10;
56
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
2030 }
57
c0f1c8be3a47 - Added code for converting hex characters to binary numbers.
aziz
parents: 56
diff changeset
2031
410
4d9ee8e60712 Added destructors for two particular tokens.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 403
diff changeset
2032 assert(ishexad(p[-1]) || p[-1] == '_' || p[-1] == 'x' || p[-1] == 'X');
389
c4bfceab7246 Applied fixes and improvements to hex float scanner.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 388
diff changeset
2033 assert(!ishexad(*p) && *p != '_');
c4bfceab7246 Applied fixes and improvements to hex float scanner.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 388
diff changeset
2034
62
96af5653acef - Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents: 61
diff changeset
2035 switch (*p)
96af5653acef - Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents: 61
diff changeset
2036 {
96af5653acef - Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents: 61
diff changeset
2037 case '.':
389
c4bfceab7246 Applied fixes and improvements to hex float scanner.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 388
diff changeset
2038 if (p[1] == '.')
62
96af5653acef - Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents: 61
diff changeset
2039 break;
389
c4bfceab7246 Applied fixes and improvements to hex float scanner.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 388
diff changeset
2040 case 'p', 'P':
c4bfceab7246 Applied fixes and improvements to hex float scanner.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 388
diff changeset
2041 return scanHexReal(t);
62
96af5653acef - Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents: 61
diff changeset
2042 default:
96af5653acef - Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents: 61
diff changeset
2043 }
389
c4bfceab7246 Applied fixes and improvements to hex float scanner.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 388
diff changeset
2044
422
ad7977fe315a Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 420
diff changeset
2045 if (digits == 0 || digits > 16)
ad7977fe315a Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 420
diff changeset
2046 error(t.start, digits == 0 ? MID.NoDigitsInHexNumber : MID.OverflowHexNumber);
389
c4bfceab7246 Applied fixes and improvements to hex float scanner.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 388
diff changeset
2047
56
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
2048 goto Lfinalize;
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
2049
396
0a4619735ce9 Applied fixes to Lexer, Parser and other classes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 393
diff changeset
2050 LscanBinary:
57
c0f1c8be3a47 - Added code for converting hex characters to binary numbers.
aziz
parents: 56
diff changeset
2051 assert(digits == 0);
410
4d9ee8e60712 Added destructors for two particular tokens.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 403
diff changeset
2052 assert(*p == 'b' || *p == 'B');
56
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
2053 while (1)
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
2054 {
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
2055 if (*++p == '0')
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
2056 {
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
2057 ++digits;
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
2058 ulong_ *= 2;
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
2059 }
367
dda55fae37de - ImportGraph.execute() can parse all modules depending on the imports of the root module.
aziz
parents: 365
diff changeset
2060 else if (*p == '1')
56
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
2061 {
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
2062 ++digits;
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
2063 ulong_ *= 2;
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
2064 ulong_ += *p - '0';
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
2065 }
367
dda55fae37de - ImportGraph.execute() can parse all modules depending on the imports of the root module.
aziz
parents: 365
diff changeset
2066 else if (*p == '_')
dda55fae37de - ImportGraph.execute() can parse all modules depending on the imports of the root module.
aziz
parents: 365
diff changeset
2067 continue;
dda55fae37de - ImportGraph.execute() can parse all modules depending on the imports of the root module.
aziz
parents: 365
diff changeset
2068 else
dda55fae37de - ImportGraph.execute() can parse all modules depending on the imports of the root module.
aziz
parents: 365
diff changeset
2069 break;
56
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
2070 }
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
2071
422
ad7977fe315a Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 420
diff changeset
2072 if (digits == 0 || digits > 64)
ad7977fe315a Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 420
diff changeset
2073 error(t.start, digits == 0 ? MID.NoDigitsInBinNumber : MID.OverflowBinaryNumber);
59
3e594725899a - Issuing error when no digits were found in hex and binary numbers.
aziz
parents: 58
diff changeset
2074
410
4d9ee8e60712 Added destructors for two particular tokens.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 403
diff changeset
2075 assert(p[-1] == '0' || p[-1] == '1' || p[-1] == '_' || p[-1] == 'b' || p[-1] == 'B', p[-1] ~ "");
367
dda55fae37de - ImportGraph.execute() can parse all modules depending on the imports of the root module.
aziz
parents: 365
diff changeset
2076 assert( !(*p == '0' || *p == '1' || *p == '_') );
56
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
2077 goto Lfinalize;
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
2078
396
0a4619735ce9 Applied fixes to Lexer, Parser and other classes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 393
diff changeset
2079 LscanOctal:
0a4619735ce9 Applied fixes to Lexer, Parser and other classes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 393
diff changeset
2080 assert(*p == '_');
62
96af5653acef - Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents: 61
diff changeset
2081 while (1)
56
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
2082 {
62
96af5653acef - Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents: 61
diff changeset
2083 if (*++p == '_')
56
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
2084 continue;
62
96af5653acef - Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents: 61
diff changeset
2085 if (!isoctal(*p))
96af5653acef - Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents: 61
diff changeset
2086 break;
396
0a4619735ce9 Applied fixes to Lexer, Parser and other classes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 393
diff changeset
2087 Loctal_enter_loop:
56
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
2088 if (ulong_ < ulong.max/2 || (ulong_ == ulong.max/2 && *p <= '1'))
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
2089 {
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
2090 ulong_ *= 8;
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
2091 ulong_ += *p - '0';
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
2092 continue;
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
2093 }
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
2094 // Overflow: skip following digits.
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
2095 overflow = true;
396
0a4619735ce9 Applied fixes to Lexer, Parser and other classes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 393
diff changeset
2096 while (isoctal(*++p)) {}
56
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
2097 break;
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
2098 }
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
2099
63
c29229fbf2f7 - Recognizing floats that start with a dot.
aziz
parents: 62
diff changeset
2100 bool hasDecimalDigits;
c29229fbf2f7 - Recognizing floats that start with a dot.
aziz
parents: 62
diff changeset
2101 if (isdigit(*p))
c29229fbf2f7 - Recognizing floats that start with a dot.
aziz
parents: 62
diff changeset
2102 {
396
0a4619735ce9 Applied fixes to Lexer, Parser and other classes.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 393
diff changeset
2103 Loctal_hasDecimalDigits:
63
c29229fbf2f7 - Recognizing floats that start with a dot.
aziz
parents: 62
diff changeset
2104 hasDecimalDigits = true;
c29229fbf2f7 - Recognizing floats that start with a dot.
aziz
parents: 62
diff changeset
2105 while (isdigit(*++p)) {}
c29229fbf2f7 - Recognizing floats that start with a dot.
aziz
parents: 62
diff changeset
2106 }
c29229fbf2f7 - Recognizing floats that start with a dot.
aziz
parents: 62
diff changeset
2107
c29229fbf2f7 - Recognizing floats that start with a dot.
aziz
parents: 62
diff changeset
2108 // The number could be a float, so check errors below.
56
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
2109 switch (*p)
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
2110 {
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
2111 case '.':
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
2112 if (p[1] != '.')
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
2113 goto LscanReal;
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
2114 break;
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
2115 case 'L':
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
2116 if (p[1] != 'i')
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
2117 break;
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
2118 case 'i', 'f', 'F', 'e', 'E':
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
2119 goto LscanReal;
63
c29229fbf2f7 - Recognizing floats that start with a dot.
aziz
parents: 62
diff changeset
2120 default:
56
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
2121 }
62
96af5653acef - Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents: 61
diff changeset
2122
63
c29229fbf2f7 - Recognizing floats that start with a dot.
aziz
parents: 62
diff changeset
2123 if (hasDecimalDigits)
422
ad7977fe315a Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 420
diff changeset
2124 error(t.start, MID.OctalNumberHasDecimals);
ad7977fe315a Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 420
diff changeset
2125
56
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
2126 if (overflow)
422
ad7977fe315a Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 420
diff changeset
2127 error(t.start, MID.OverflowOctalNumber);
56
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
2128 // goto Lfinalize;
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
2129
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
2130 Lfinalize:
58
50bb7fc9db44 - The types of integers are recognized now.
aziz
parents: 57
diff changeset
2131 enum Suffix
50bb7fc9db44 - The types of integers are recognized now.
aziz
parents: 57
diff changeset
2132 {
50bb7fc9db44 - The types of integers are recognized now.
aziz
parents: 57
diff changeset
2133 None = 0,
50bb7fc9db44 - The types of integers are recognized now.
aziz
parents: 57
diff changeset
2134 Unsigned = 1,
60
32cc23bd217b - Fixed number suffix scanning.
aziz
parents: 59
diff changeset
2135 Long = 2
58
50bb7fc9db44 - The types of integers are recognized now.
aziz
parents: 57
diff changeset
2136 }
60
32cc23bd217b - Fixed number suffix scanning.
aziz
parents: 59
diff changeset
2137
392
bb935c6f9b7a Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 391
diff changeset
2138 // Scan optional suffix: L, Lu, LU, u, uL, U or UL.
58
50bb7fc9db44 - The types of integers are recognized now.
aziz
parents: 57
diff changeset
2139 Suffix suffix;
56
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
2140 while (1)
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
2141 {
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
2142 switch (*p)
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
2143 {
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
2144 case 'L':
58
50bb7fc9db44 - The types of integers are recognized now.
aziz
parents: 57
diff changeset
2145 if (suffix & Suffix.Long)
56
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
2146 break;
60
32cc23bd217b - Fixed number suffix scanning.
aziz
parents: 59
diff changeset
2147 suffix |= Suffix.Long;
32cc23bd217b - Fixed number suffix scanning.
aziz
parents: 59
diff changeset
2148 ++p;
56
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
2149 continue;
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
2150 case 'u', 'U':
58
50bb7fc9db44 - The types of integers are recognized now.
aziz
parents: 57
diff changeset
2151 if (suffix & Suffix.Unsigned)
56
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
2152 break;
60
32cc23bd217b - Fixed number suffix scanning.
aziz
parents: 59
diff changeset
2153 suffix |= Suffix.Unsigned;
32cc23bd217b - Fixed number suffix scanning.
aziz
parents: 59
diff changeset
2154 ++p;
32cc23bd217b - Fixed number suffix scanning.
aziz
parents: 59
diff changeset
2155 continue;
58
50bb7fc9db44 - The types of integers are recognized now.
aziz
parents: 57
diff changeset
2156 default:
50bb7fc9db44 - The types of integers are recognized now.
aziz
parents: 57
diff changeset
2157 break;
56
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
2158 }
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
2159 break;
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
2160 }
58
50bb7fc9db44 - The types of integers are recognized now.
aziz
parents: 57
diff changeset
2161
392
bb935c6f9b7a Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 391
diff changeset
2162 // Determine type of Integer.
58
50bb7fc9db44 - The types of integers are recognized now.
aziz
parents: 57
diff changeset
2163 switch (suffix)
50bb7fc9db44 - The types of integers are recognized now.
aziz
parents: 57
diff changeset
2164 {
50bb7fc9db44 - The types of integers are recognized now.
aziz
parents: 57
diff changeset
2165 case Suffix.None:
422
ad7977fe315a Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 420
diff changeset
2166 if (ulong_ & 0x8000_0000_0000_0000)
58
50bb7fc9db44 - The types of integers are recognized now.
aziz
parents: 57
diff changeset
2167 {
50bb7fc9db44 - The types of integers are recognized now.
aziz
parents: 57
diff changeset
2168 if (isDecimal)
422
ad7977fe315a Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 420
diff changeset
2169 error(t.start, MID.OverflowDecimalSign);
58
50bb7fc9db44 - The types of integers are recognized now.
aziz
parents: 57
diff changeset
2170 t.type = TOK.Uint64;
50bb7fc9db44 - The types of integers are recognized now.
aziz
parents: 57
diff changeset
2171 }
422
ad7977fe315a Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 420
diff changeset
2172 else if (ulong_ & 0xFFFF_FFFF_0000_0000)
58
50bb7fc9db44 - The types of integers are recognized now.
aziz
parents: 57
diff changeset
2173 t.type = TOK.Int64;
422
ad7977fe315a Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 420
diff changeset
2174 else if (ulong_ & 0x8000_0000)
58
50bb7fc9db44 - The types of integers are recognized now.
aziz
parents: 57
diff changeset
2175 t.type = isDecimal ? TOK.Int64 : TOK.Uint32;
50bb7fc9db44 - The types of integers are recognized now.
aziz
parents: 57
diff changeset
2176 else
50bb7fc9db44 - The types of integers are recognized now.
aziz
parents: 57
diff changeset
2177 t.type = TOK.Int32;
50bb7fc9db44 - The types of integers are recognized now.
aziz
parents: 57
diff changeset
2178 break;
50bb7fc9db44 - The types of integers are recognized now.
aziz
parents: 57
diff changeset
2179 case Suffix.Unsigned:
422
ad7977fe315a Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 420
diff changeset
2180 if (ulong_ & 0xFFFF_FFFF_0000_0000)
58
50bb7fc9db44 - The types of integers are recognized now.
aziz
parents: 57
diff changeset
2181 t.type = TOK.Uint64;
50bb7fc9db44 - The types of integers are recognized now.
aziz
parents: 57
diff changeset
2182 else
50bb7fc9db44 - The types of integers are recognized now.
aziz
parents: 57
diff changeset
2183 t.type = TOK.Uint32;
50bb7fc9db44 - The types of integers are recognized now.
aziz
parents: 57
diff changeset
2184 break;
50bb7fc9db44 - The types of integers are recognized now.
aziz
parents: 57
diff changeset
2185 case Suffix.Long:
422
ad7977fe315a Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 420
diff changeset
2186 if (ulong_ & 0x8000_0000_0000_0000)
58
50bb7fc9db44 - The types of integers are recognized now.
aziz
parents: 57
diff changeset
2187 {
50bb7fc9db44 - The types of integers are recognized now.
aziz
parents: 57
diff changeset
2188 if (isDecimal)
422
ad7977fe315a Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 420
diff changeset
2189 error(t.start, MID.OverflowDecimalSign);
58
50bb7fc9db44 - The types of integers are recognized now.
aziz
parents: 57
diff changeset
2190 t.type = TOK.Uint64;
50bb7fc9db44 - The types of integers are recognized now.
aziz
parents: 57
diff changeset
2191 }
50bb7fc9db44 - The types of integers are recognized now.
aziz
parents: 57
diff changeset
2192 else
50bb7fc9db44 - The types of integers are recognized now.
aziz
parents: 57
diff changeset
2193 t.type = TOK.Int64;
50bb7fc9db44 - The types of integers are recognized now.
aziz
parents: 57
diff changeset
2194 break;
50bb7fc9db44 - The types of integers are recognized now.
aziz
parents: 57
diff changeset
2195 case Suffix.Unsigned | Suffix.Long:
50bb7fc9db44 - The types of integers are recognized now.
aziz
parents: 57
diff changeset
2196 t.type = TOK.Uint64;
50bb7fc9db44 - The types of integers are recognized now.
aziz
parents: 57
diff changeset
2197 break;
50bb7fc9db44 - The types of integers are recognized now.
aziz
parents: 57
diff changeset
2198 default:
50bb7fc9db44 - The types of integers are recognized now.
aziz
parents: 57
diff changeset
2199 assert(0);
50bb7fc9db44 - The types of integers are recognized now.
aziz
parents: 57
diff changeset
2200 }
56
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
2201 t.ulong_ = ulong_;
15
c70c028e47dd - Started implementation of lexing numbers.
aziz
parents: 14
diff changeset
2202 t.end = p;
56
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
2203 return;
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
2204 LscanReal:
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
2205 scanReal(t);
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
2206 return;
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
2207 }
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
2208
63
c29229fbf2f7 - Recognizing floats that start with a dot.
aziz
parents: 62
diff changeset
2209 /*
c29229fbf2f7 - Recognizing floats that start with a dot.
aziz
parents: 62
diff changeset
2210 FloatLiteral:= Float[fFL]?i?
c29229fbf2f7 - Recognizing floats that start with a dot.
aziz
parents: 62
diff changeset
2211 Float:= DecFloat | HexFloat
67
996065105910 - Fix: variadic arguments are local to the variadic function. Parameters are converted to an array of strings first and then passed to the constructor of class Problem.
aziz
parents: 66
diff changeset
2212 DecFloat:= ([0-9][0-9_]*[.][0-9_]*DecExponent?) | [.][0-9][0-9_]*DecExponent? | [0-9][0-9_]*DecExponent
63
c29229fbf2f7 - Recognizing floats that start with a dot.
aziz
parents: 62
diff changeset
2213 DecExponent:= [eE][+-]?[0-9][0-9_]*
c29229fbf2f7 - Recognizing floats that start with a dot.
aziz
parents: 62
diff changeset
2214 HexFloat:= 0[xX](HexDigits[.]HexDigits | [.][0-9a-zA-Z]HexDigits? | HexDigits)HexExponent
c29229fbf2f7 - Recognizing floats that start with a dot.
aziz
parents: 62
diff changeset
2215 HexExponent:= [pP][+-]?[0-9][0-9_]*
c29229fbf2f7 - Recognizing floats that start with a dot.
aziz
parents: 62
diff changeset
2216 */
56
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
2217 void scanReal(ref Token t)
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 55
diff changeset
2218 {
63
c29229fbf2f7 - Recognizing floats that start with a dot.
aziz
parents: 62
diff changeset
2219 if (*p == '.')
392
bb935c6f9b7a Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 391
diff changeset
2220 {
bb935c6f9b7a Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 391
diff changeset
2221 assert(p[1] != '.');
63
c29229fbf2f7 - Recognizing floats that start with a dot.
aziz
parents: 62
diff changeset
2222 // This function was called by scan() or scanNumber().
c29229fbf2f7 - Recognizing floats that start with a dot.
aziz
parents: 62
diff changeset
2223 while (isdigit(*++p) || *p == '_') {}
392
bb935c6f9b7a Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 391
diff changeset
2224 }
63
c29229fbf2f7 - Recognizing floats that start with a dot.
aziz
parents: 62
diff changeset
2225 else
c29229fbf2f7 - Recognizing floats that start with a dot.
aziz
parents: 62
diff changeset
2226 // This function was called by scanNumber().
422
ad7977fe315a Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 420
diff changeset
2227 assert(delegate ()
392
bb935c6f9b7a Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 391
diff changeset
2228 {
422
ad7977fe315a Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 420
diff changeset
2229 switch (*p)
ad7977fe315a Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 420
diff changeset
2230 {
ad7977fe315a Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 420
diff changeset
2231 case 'L':
ad7977fe315a Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 420
diff changeset
2232 if (p[1] != 'i')
ad7977fe315a Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 420
diff changeset
2233 return false;
ad7977fe315a Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 420
diff changeset
2234 case 'i', 'f', 'F', 'e', 'E':
ad7977fe315a Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 420
diff changeset
2235 return true;
ad7977fe315a Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 420
diff changeset
2236 default:
ad7977fe315a Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 420
diff changeset
2237 }
ad7977fe315a Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 420
diff changeset
2238 return false;
ad7977fe315a Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 420
diff changeset
2239 }()
392
bb935c6f9b7a Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 391
diff changeset
2240 );
63
c29229fbf2f7 - Recognizing floats that start with a dot.
aziz
parents: 62
diff changeset
2241
c29229fbf2f7 - Recognizing floats that start with a dot.
aziz
parents: 62
diff changeset
2242 // Scan exponent.
c29229fbf2f7 - Recognizing floats that start with a dot.
aziz
parents: 62
diff changeset
2243 if (*p == 'e' || *p == 'E')
c29229fbf2f7 - Recognizing floats that start with a dot.
aziz
parents: 62
diff changeset
2244 {
c29229fbf2f7 - Recognizing floats that start with a dot.
aziz
parents: 62
diff changeset
2245 ++p;
c29229fbf2f7 - Recognizing floats that start with a dot.
aziz
parents: 62
diff changeset
2246 if (*p == '-' || *p == '+')
c29229fbf2f7 - Recognizing floats that start with a dot.
aziz
parents: 62
diff changeset
2247 ++p;
422
ad7977fe315a Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 420
diff changeset
2248 if (isdigit(*p))
ad7977fe315a Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 420
diff changeset
2249 while (isdigit(*++p) || *p == '_') {}
63
c29229fbf2f7 - Recognizing floats that start with a dot.
aziz
parents: 62
diff changeset
2250 else
422
ad7977fe315a Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 420
diff changeset
2251 error(t.start, MID.FloatExpMustStartWithDigit);
63
c29229fbf2f7 - Recognizing floats that start with a dot.
aziz
parents: 62
diff changeset
2252 }
c29229fbf2f7 - Recognizing floats that start with a dot.
aziz
parents: 62
diff changeset
2253
392
bb935c6f9b7a Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 391
diff changeset
2254 // Copy whole number and remove underscores from buffer.
bb935c6f9b7a Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 391
diff changeset
2255 char[] buffer = t.start[0..p-t.start].dup;
bb935c6f9b7a Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 391
diff changeset
2256 uint j;
bb935c6f9b7a Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 391
diff changeset
2257 foreach (c; buffer)
bb935c6f9b7a Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 391
diff changeset
2258 if (c != '_')
bb935c6f9b7a Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 391
diff changeset
2259 buffer[j++] = c;
bb935c6f9b7a Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 391
diff changeset
2260 buffer.length = j; // Adjust length.
bb935c6f9b7a Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 391
diff changeset
2261 buffer ~= 0; // Terminate for C functions.
bb935c6f9b7a Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 391
diff changeset
2262
63
c29229fbf2f7 - Recognizing floats that start with a dot.
aziz
parents: 62
diff changeset
2263 finalizeFloat(t, buffer);
30
426767b94635 - Added code for parsing the '#line' special token.
aziz
parents: 29
diff changeset
2264 }
426767b94635 - Added code for parsing the '#line' special token.
aziz
parents: 29
diff changeset
2265
62
96af5653acef - Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents: 61
diff changeset
2266 void scanHexReal(ref Token t)
96af5653acef - Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents: 61
diff changeset
2267 {
389
c4bfceab7246 Applied fixes and improvements to hex float scanner.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 388
diff changeset
2268 assert(*p == '.' || *p == 'p' || *p == 'P');
62
96af5653acef - Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents: 61
diff changeset
2269 MID mid;
96af5653acef - Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents: 61
diff changeset
2270 if (*p == '.')
389
c4bfceab7246 Applied fixes and improvements to hex float scanner.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 388
diff changeset
2271 while (ishexad(*++p) || *p == '_')
c4bfceab7246 Applied fixes and improvements to hex float scanner.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 388
diff changeset
2272 {}
c4bfceab7246 Applied fixes and improvements to hex float scanner.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 388
diff changeset
2273 // Decimal exponent is required.
62
96af5653acef - Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents: 61
diff changeset
2274 if (*p != 'p' && *p != 'P')
96af5653acef - Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents: 61
diff changeset
2275 {
96af5653acef - Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents: 61
diff changeset
2276 mid = MID.HexFloatExponentRequired;
96af5653acef - Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents: 61
diff changeset
2277 goto Lerr;
96af5653acef - Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents: 61
diff changeset
2278 }
389
c4bfceab7246 Applied fixes and improvements to hex float scanner.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 388
diff changeset
2279 // Scan exponent
c4bfceab7246 Applied fixes and improvements to hex float scanner.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 388
diff changeset
2280 assert(*p == 'p' || *p == 'P');
393
fce1e6133dac Applied fix to hex float scanner.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 392
diff changeset
2281 ++p;
fce1e6133dac Applied fix to hex float scanner.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 392
diff changeset
2282 if (*p == '+' || *p == '-')
fce1e6133dac Applied fix to hex float scanner.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 392
diff changeset
2283 ++p;
fce1e6133dac Applied fix to hex float scanner.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 392
diff changeset
2284 if (!isdigit(*p))
62
96af5653acef - Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents: 61
diff changeset
2285 {
389
c4bfceab7246 Applied fixes and improvements to hex float scanner.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 388
diff changeset
2286 mid = MID.HexFloatExpMustStartWithDigit;
62
96af5653acef - Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents: 61
diff changeset
2287 goto Lerr;
96af5653acef - Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents: 61
diff changeset
2288 }
389
c4bfceab7246 Applied fixes and improvements to hex float scanner.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 388
diff changeset
2289 while (isdigit(*++p) || *p == '_')
c4bfceab7246 Applied fixes and improvements to hex float scanner.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 388
diff changeset
2290 {}
c4bfceab7246 Applied fixes and improvements to hex float scanner.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 388
diff changeset
2291 // Copy whole number and remove underscores from buffer.
c4bfceab7246 Applied fixes and improvements to hex float scanner.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 388
diff changeset
2292 char[] buffer = t.start[0..p-t.start].dup;
c4bfceab7246 Applied fixes and improvements to hex float scanner.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 388
diff changeset
2293 uint j;
c4bfceab7246 Applied fixes and improvements to hex float scanner.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 388
diff changeset
2294 foreach (c; buffer)
c4bfceab7246 Applied fixes and improvements to hex float scanner.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 388
diff changeset
2295 if (c != '_')
c4bfceab7246 Applied fixes and improvements to hex float scanner.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 388
diff changeset
2296 buffer[j++] = c;
c4bfceab7246 Applied fixes and improvements to hex float scanner.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 388
diff changeset
2297 buffer.length = j; // Adjust length.
62
96af5653acef - Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents: 61
diff changeset
2298 buffer ~= 0; // Terminate for C functions.
63
c29229fbf2f7 - Recognizing floats that start with a dot.
aziz
parents: 62
diff changeset
2299 finalizeFloat(t, buffer);
c29229fbf2f7 - Recognizing floats that start with a dot.
aziz
parents: 62
diff changeset
2300 return;
c29229fbf2f7 - Recognizing floats that start with a dot.
aziz
parents: 62
diff changeset
2301 Lerr:
c29229fbf2f7 - Recognizing floats that start with a dot.
aziz
parents: 62
diff changeset
2302 t.type = TOK.Float32;
c29229fbf2f7 - Recognizing floats that start with a dot.
aziz
parents: 62
diff changeset
2303 t.end = p;
422
ad7977fe315a Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 420
diff changeset
2304 error(t.start, mid);
63
c29229fbf2f7 - Recognizing floats that start with a dot.
aziz
parents: 62
diff changeset
2305 }
c29229fbf2f7 - Recognizing floats that start with a dot.
aziz
parents: 62
diff changeset
2306
c29229fbf2f7 - Recognizing floats that start with a dot.
aziz
parents: 62
diff changeset
2307 void finalizeFloat(ref Token t, string buffer)
c29229fbf2f7 - Recognizing floats that start with a dot.
aziz
parents: 62
diff changeset
2308 {
389
c4bfceab7246 Applied fixes and improvements to hex float scanner.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 388
diff changeset
2309 assert(buffer[$-1] == 0);
62
96af5653acef - Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents: 61
diff changeset
2310 // Float number is well-formed. Check suffixes and do conversion.
96af5653acef - Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents: 61
diff changeset
2311 switch (*p)
96af5653acef - Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents: 61
diff changeset
2312 {
96af5653acef - Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents: 61
diff changeset
2313 case 'f', 'F':
96af5653acef - Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents: 61
diff changeset
2314 t.type = TOK.Float32;
96af5653acef - Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents: 61
diff changeset
2315 t.float_ = strtof(buffer.ptr, null);
96af5653acef - Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents: 61
diff changeset
2316 ++p;
96af5653acef - Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents: 61
diff changeset
2317 break;
96af5653acef - Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents: 61
diff changeset
2318 case 'L':
96af5653acef - Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents: 61
diff changeset
2319 t.type = TOK.Float80;
96af5653acef - Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents: 61
diff changeset
2320 t.real_ = strtold(buffer.ptr, null);
96af5653acef - Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents: 61
diff changeset
2321 ++p;
96af5653acef - Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents: 61
diff changeset
2322 break;
96af5653acef - Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents: 61
diff changeset
2323 default:
96af5653acef - Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents: 61
diff changeset
2324 t.type = TOK.Float64;
96af5653acef - Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents: 61
diff changeset
2325 t.double_ = strtod(buffer.ptr, null);
96af5653acef - Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents: 61
diff changeset
2326 break;
96af5653acef - Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents: 61
diff changeset
2327 }
96af5653acef - Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents: 61
diff changeset
2328 if (*p == 'i')
96af5653acef - Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents: 61
diff changeset
2329 {
96af5653acef - Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents: 61
diff changeset
2330 ++p;
63
c29229fbf2f7 - Recognizing floats that start with a dot.
aziz
parents: 62
diff changeset
2331 t.type += 3; // Switch to imaginary counterpart.
62
96af5653acef - Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents: 61
diff changeset
2332 }
391
33b566df6af4 Migrated project to Tango.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 390
diff changeset
2333 if (errno() == ERANGE)
422
ad7977fe315a Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 420
diff changeset
2334 error(t.start, MID.OverflowFloatNumber);
62
96af5653acef - Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents: 61
diff changeset
2335 t.end = p;
96af5653acef - Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents: 61
diff changeset
2336 }
96af5653acef - Fixed loop of hex number scanner. Moved checks under the switch block.
aziz
parents: 61
diff changeset
2337
30
426767b94635 - Added code for parsing the '#line' special token.
aziz
parents: 29
diff changeset
2338 /// Scan special token: #line Integer [Filespec] EndOfLine
360
b6a3755eba94 - Renamed scanSpecialToken() to scanSpecialTokenSequence().
aziz
parents: 350
diff changeset
2339 void scanSpecialTokenSequence(ref Token t)
30
426767b94635 - Added code for parsing the '#line' special token.
aziz
parents: 29
diff changeset
2340 {
47
8aa37a78937b - Properly implemented scanner for normal string literals.
aziz
parents: 46
diff changeset
2341 assert(*p == '#');
323
6259fb93e3dd - Rewrote scanSpecialToken().
aziz
parents: 315
diff changeset
2342 t.type = TOK.HashLine;
6259fb93e3dd - Rewrote scanSpecialToken().
aziz
parents: 315
diff changeset
2343
30
426767b94635 - Added code for parsing the '#line' special token.
aziz
parents: 29
diff changeset
2344 MID mid;
422
ad7977fe315a Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 420
diff changeset
2345 auto errorAtColumn = p;
30
426767b94635 - Added code for parsing the '#line' special token.
aziz
parents: 29
diff changeset
2346
323
6259fb93e3dd - Rewrote scanSpecialToken().
aziz
parents: 315
diff changeset
2347 ++p;
6259fb93e3dd - Rewrote scanSpecialToken().
aziz
parents: 315
diff changeset
2348 if (p[0] != 'l' || p[1] != 'i' || p[2] != 'n' || p[3] != 'e')
51
cadd2bfe686c - Displaying error messages in XML.
aziz
parents: 50
diff changeset
2349 {
370
ae4afb66768f - Renamed findModule() to findModulePath().
aziz
parents: 367
diff changeset
2350 mid = MID.ExpectedIdentifierSTLine;
51
cadd2bfe686c - Displaying error messages in XML.
aziz
parents: 50
diff changeset
2351 goto Lerr;
cadd2bfe686c - Displaying error messages in XML.
aziz
parents: 50
diff changeset
2352 }
323
6259fb93e3dd - Rewrote scanSpecialToken().
aziz
parents: 315
diff changeset
2353 p += 3;
6259fb93e3dd - Rewrote scanSpecialToken().
aziz
parents: 315
diff changeset
2354
388
ae154eceba65 Applied some fixes to scanning and printing #line tokens.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 387
diff changeset
2355 // TODO: #line58"path/file" is legal. Require spaces?
ae154eceba65 Applied some fixes to scanning and printing #line tokens.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 387
diff changeset
2356 // State.Space could be used for that purpose.
323
6259fb93e3dd - Rewrote scanSpecialToken().
aziz
parents: 315
diff changeset
2357 enum State
388
ae154eceba65 Applied some fixes to scanning and printing #line tokens.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 387
diff changeset
2358 { /+Space,+/ Integer, Filespec, End }
51
cadd2bfe686c - Displaying error messages in XML.
aziz
parents: 50
diff changeset
2359
388
ae154eceba65 Applied some fixes to scanning and printing #line tokens.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 387
diff changeset
2360 State state = State.Integer;
323
6259fb93e3dd - Rewrote scanSpecialToken().
aziz
parents: 315
diff changeset
2361
6259fb93e3dd - Rewrote scanSpecialToken().
aziz
parents: 315
diff changeset
2362 Loop:
6259fb93e3dd - Rewrote scanSpecialToken().
aziz
parents: 315
diff changeset
2363 while (1)
51
cadd2bfe686c - Displaying error messages in XML.
aziz
parents: 50
diff changeset
2364 {
323
6259fb93e3dd - Rewrote scanSpecialToken().
aziz
parents: 315
diff changeset
2365 switch (*++p)
6259fb93e3dd - Rewrote scanSpecialToken().
aziz
parents: 315
diff changeset
2366 {
403
e7228859d865 Fixed the way newlines are handled after #! and #line tokens.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 402
diff changeset
2367 case LS[0]:
e7228859d865 Fixed the way newlines are handled after #! and #line tokens.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 402
diff changeset
2368 if (!(p[1] == LS[1] && (p[2] == LS[2] || p[2] == PS[2])))
e7228859d865 Fixed the way newlines are handled after #! and #line tokens.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 402
diff changeset
2369 goto default;
e7228859d865 Fixed the way newlines are handled after #! and #line tokens.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 402
diff changeset
2370 case '\r', '\n', 0, _Z_:
323
6259fb93e3dd - Rewrote scanSpecialToken().
aziz
parents: 315
diff changeset
2371 break Loop;
6259fb93e3dd - Rewrote scanSpecialToken().
aziz
parents: 315
diff changeset
2372 default:
6259fb93e3dd - Rewrote scanSpecialToken().
aziz
parents: 315
diff changeset
2373 if (isspace(*p))
6259fb93e3dd - Rewrote scanSpecialToken().
aziz
parents: 315
diff changeset
2374 continue;
388
ae154eceba65 Applied some fixes to scanning and printing #line tokens.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 387
diff changeset
2375 if (state == State.Integer)
323
6259fb93e3dd - Rewrote scanSpecialToken().
aziz
parents: 315
diff changeset
2376 {
6259fb93e3dd - Rewrote scanSpecialToken().
aziz
parents: 315
diff changeset
2377 if (!isdigit(*p))
6259fb93e3dd - Rewrote scanSpecialToken().
aziz
parents: 315
diff changeset
2378 {
422
ad7977fe315a Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 420
diff changeset
2379 errorAtColumn = p;
388
ae154eceba65 Applied some fixes to scanning and printing #line tokens.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 387
diff changeset
2380 mid = MID.ExpectedIntegerAfterSTLine;
323
6259fb93e3dd - Rewrote scanSpecialToken().
aziz
parents: 315
diff changeset
2381 goto Lerr;
6259fb93e3dd - Rewrote scanSpecialToken().
aziz
parents: 315
diff changeset
2382 }
6259fb93e3dd - Rewrote scanSpecialToken().
aziz
parents: 315
diff changeset
2383 t.line_num = new Token;
6259fb93e3dd - Rewrote scanSpecialToken().
aziz
parents: 315
diff changeset
2384 scan(*t.line_num);
388
ae154eceba65 Applied some fixes to scanning and printing #line tokens.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 387
diff changeset
2385 if (t.line_num.type != TOK.Int32 && t.line_num.type != TOK.Uint32)
ae154eceba65 Applied some fixes to scanning and printing #line tokens.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 387
diff changeset
2386 {
422
ad7977fe315a Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 420
diff changeset
2387 errorAtColumn = t.line_num.start;
388
ae154eceba65 Applied some fixes to scanning and printing #line tokens.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 387
diff changeset
2388 mid = MID.ExpectedIntegerAfterSTLine;
ae154eceba65 Applied some fixes to scanning and printing #line tokens.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 387
diff changeset
2389 goto Lerr;
ae154eceba65 Applied some fixes to scanning and printing #line tokens.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 387
diff changeset
2390 }
389
c4bfceab7246 Applied fixes and improvements to hex float scanner.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 388
diff changeset
2391 --p; // Go one back because scan() advanced p past the integer.
323
6259fb93e3dd - Rewrote scanSpecialToken().
aziz
parents: 315
diff changeset
2392 state = State.Filespec;
6259fb93e3dd - Rewrote scanSpecialToken().
aziz
parents: 315
diff changeset
2393 }
6259fb93e3dd - Rewrote scanSpecialToken().
aziz
parents: 315
diff changeset
2394 else if (state == State.Filespec)
6259fb93e3dd - Rewrote scanSpecialToken().
aziz
parents: 315
diff changeset
2395 {
6259fb93e3dd - Rewrote scanSpecialToken().
aziz
parents: 315
diff changeset
2396 if (*p != '"')
6259fb93e3dd - Rewrote scanSpecialToken().
aziz
parents: 315
diff changeset
2397 {
422
ad7977fe315a Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 420
diff changeset
2398 errorAtColumn = p;
323
6259fb93e3dd - Rewrote scanSpecialToken().
aziz
parents: 315
diff changeset
2399 mid = MID.ExpectedFilespec;
6259fb93e3dd - Rewrote scanSpecialToken().
aziz
parents: 315
diff changeset
2400 goto Lerr;
6259fb93e3dd - Rewrote scanSpecialToken().
aziz
parents: 315
diff changeset
2401 }
6259fb93e3dd - Rewrote scanSpecialToken().
aziz
parents: 315
diff changeset
2402 t.line_filespec = new Token;
6259fb93e3dd - Rewrote scanSpecialToken().
aziz
parents: 315
diff changeset
2403 t.line_filespec.start = p;
6259fb93e3dd - Rewrote scanSpecialToken().
aziz
parents: 315
diff changeset
2404 t.line_filespec.type = TOK.Filespec;
6259fb93e3dd - Rewrote scanSpecialToken().
aziz
parents: 315
diff changeset
2405 while (1)
6259fb93e3dd - Rewrote scanSpecialToken().
aziz
parents: 315
diff changeset
2406 {
6259fb93e3dd - Rewrote scanSpecialToken().
aziz
parents: 315
diff changeset
2407 switch (*++p)
6259fb93e3dd - Rewrote scanSpecialToken().
aziz
parents: 315
diff changeset
2408 {
6259fb93e3dd - Rewrote scanSpecialToken().
aziz
parents: 315
diff changeset
2409 case '"':
6259fb93e3dd - Rewrote scanSpecialToken().
aziz
parents: 315
diff changeset
2410 break;
6259fb93e3dd - Rewrote scanSpecialToken().
aziz
parents: 315
diff changeset
2411 case LS[0]:
6259fb93e3dd - Rewrote scanSpecialToken().
aziz
parents: 315
diff changeset
2412 if (!(p[1] == LS[1] && (p[2] == LS[2] || p[2] == PS[2])))
6259fb93e3dd - Rewrote scanSpecialToken().
aziz
parents: 315
diff changeset
2413 goto default;
6259fb93e3dd - Rewrote scanSpecialToken().
aziz
parents: 315
diff changeset
2414 case '\r', '\n', 0, _Z_:
422
ad7977fe315a Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 420
diff changeset
2415 errorAtColumn = t.line_filespec.start;
323
6259fb93e3dd - Rewrote scanSpecialToken().
aziz
parents: 315
diff changeset
2416 mid = MID.UnterminatedFilespec;
6259fb93e3dd - Rewrote scanSpecialToken().
aziz
parents: 315
diff changeset
2417 t.line_filespec.end = p;
6259fb93e3dd - Rewrote scanSpecialToken().
aziz
parents: 315
diff changeset
2418 goto Lerr;
6259fb93e3dd - Rewrote scanSpecialToken().
aziz
parents: 315
diff changeset
2419 default:
6259fb93e3dd - Rewrote scanSpecialToken().
aziz
parents: 315
diff changeset
2420 if (*p & 128)
6259fb93e3dd - Rewrote scanSpecialToken().
aziz
parents: 315
diff changeset
2421 decodeUTF8();
6259fb93e3dd - Rewrote scanSpecialToken().
aziz
parents: 315
diff changeset
2422 continue;
6259fb93e3dd - Rewrote scanSpecialToken().
aziz
parents: 315
diff changeset
2423 }
6259fb93e3dd - Rewrote scanSpecialToken().
aziz
parents: 315
diff changeset
2424 break; // Exit loop.
6259fb93e3dd - Rewrote scanSpecialToken().
aziz
parents: 315
diff changeset
2425 }
6259fb93e3dd - Rewrote scanSpecialToken().
aziz
parents: 315
diff changeset
2426 auto start = t.line_filespec.start +1; // +1 skips '"'
6259fb93e3dd - Rewrote scanSpecialToken().
aziz
parents: 315
diff changeset
2427 t.line_filespec.str = start[0 .. p - start];
6259fb93e3dd - Rewrote scanSpecialToken().
aziz
parents: 315
diff changeset
2428 t.line_filespec.end = p + 1;
6259fb93e3dd - Rewrote scanSpecialToken().
aziz
parents: 315
diff changeset
2429 state = State.End;
6259fb93e3dd - Rewrote scanSpecialToken().
aziz
parents: 315
diff changeset
2430 }
6259fb93e3dd - Rewrote scanSpecialToken().
aziz
parents: 315
diff changeset
2431 else/+ if (state == State.End)+/
6259fb93e3dd - Rewrote scanSpecialToken().
aziz
parents: 315
diff changeset
2432 {
6259fb93e3dd - Rewrote scanSpecialToken().
aziz
parents: 315
diff changeset
2433 mid = MID.UnterminatedSpecialToken;
6259fb93e3dd - Rewrote scanSpecialToken().
aziz
parents: 315
diff changeset
2434 goto Lerr;
6259fb93e3dd - Rewrote scanSpecialToken().
aziz
parents: 315
diff changeset
2435 }
6259fb93e3dd - Rewrote scanSpecialToken().
aziz
parents: 315
diff changeset
2436 }
51
cadd2bfe686c - Displaying error messages in XML.
aziz
parents: 50
diff changeset
2437 }
403
e7228859d865 Fixed the way newlines are handled after #! and #line tokens.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 402
diff changeset
2438 assert(*p == '\r' || *p == '\n' || *p == 0 || *p == _Z_ ||
e7228859d865 Fixed the way newlines are handled after #! and #line tokens.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 402
diff changeset
2439 *p == LS[0] && (p[1] == LS[1] && (p[2] == LS[2] || p[2] == PS[2]))
e7228859d865 Fixed the way newlines are handled after #! and #line tokens.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 402
diff changeset
2440 );
323
6259fb93e3dd - Rewrote scanSpecialToken().
aziz
parents: 315
diff changeset
2441
388
ae154eceba65 Applied some fixes to scanning and printing #line tokens.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 387
diff changeset
2442 if (state == State.Integer)
51
cadd2bfe686c - Displaying error messages in XML.
aziz
parents: 50
diff changeset
2443 {
422
ad7977fe315a Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 420
diff changeset
2444 errorAtColumn = p;
388
ae154eceba65 Applied some fixes to scanning and printing #line tokens.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 387
diff changeset
2445 mid = MID.ExpectedIntegerAfterSTLine;
30
426767b94635 - Added code for parsing the '#line' special token.
aziz
parents: 29
diff changeset
2446 goto Lerr;
426767b94635 - Added code for parsing the '#line' special token.
aziz
parents: 29
diff changeset
2447 }
426767b94635 - Added code for parsing the '#line' special token.
aziz
parents: 29
diff changeset
2448
392
bb935c6f9b7a Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 391
diff changeset
2449 // Evaluate #line only when not in token string.
bb935c6f9b7a Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 391
diff changeset
2450 if (!inTokenString)
bb935c6f9b7a Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 391
diff changeset
2451 {
bb935c6f9b7a Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 391
diff changeset
2452 this.loc_old = this.loc;
bb935c6f9b7a Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 391
diff changeset
2453 this.loc_hline = t.line_num.uint_ - 1;
bb935c6f9b7a Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 391
diff changeset
2454 if (t.line_filespec)
422
ad7977fe315a Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 420
diff changeset
2455 this.errorLoc.setFilePath(t.line_filespec.str);
392
bb935c6f9b7a Applied fixes and improvements to the Lexer class.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 391
diff changeset
2456 }
323
6259fb93e3dd - Rewrote scanSpecialToken().
aziz
parents: 315
diff changeset
2457 t.end = p;
30
426767b94635 - Added code for parsing the '#line' special token.
aziz
parents: 29
diff changeset
2458
426767b94635 - Added code for parsing the '#line' special token.
aziz
parents: 29
diff changeset
2459 return;
426767b94635 - Added code for parsing the '#line' special token.
aziz
parents: 29
diff changeset
2460 Lerr:
323
6259fb93e3dd - Rewrote scanSpecialToken().
aziz
parents: 315
diff changeset
2461 t.end = p;
422
ad7977fe315a Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 420
diff changeset
2462 error(errorAtColumn, mid);
371
01887f05d4b0 - Added members loc_old and loc_hline to class Lexer.
aziz
parents: 370
diff changeset
2463 }
01887f05d4b0 - Added members loc_old and loc_hline to class Lexer.
aziz
parents: 370
diff changeset
2464
414
9c69615a4876 Added method insertEmptyTokenBefore() to Lexer. Made some fixes, too.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 413
diff changeset
2465 /+
9c69615a4876 Added method insertEmptyTokenBefore() to Lexer. Made some fixes, too.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 413
diff changeset
2466 Insert an empty dummy token before t.
9c69615a4876 Added method insertEmptyTokenBefore() to Lexer. Made some fixes, too.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 413
diff changeset
2467 Useful in the parsing phase for representing a node in the AST
9c69615a4876 Added method insertEmptyTokenBefore() to Lexer. Made some fixes, too.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 413
diff changeset
2468 that doesn't consume an actual token from the source text.
9c69615a4876 Added method insertEmptyTokenBefore() to Lexer. Made some fixes, too.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 413
diff changeset
2469 +/
9c69615a4876 Added method insertEmptyTokenBefore() to Lexer. Made some fixes, too.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 413
diff changeset
2470 Token* insertEmptyTokenBefore(Token* t)
9c69615a4876 Added method insertEmptyTokenBefore() to Lexer. Made some fixes, too.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 413
diff changeset
2471 {
9c69615a4876 Added method insertEmptyTokenBefore() to Lexer. Made some fixes, too.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 413
diff changeset
2472 assert(t !is null && t.prev !is null);
9c69615a4876 Added method insertEmptyTokenBefore() to Lexer. Made some fixes, too.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 413
diff changeset
2473 assert(text.ptr <= t.start && t.start < end, Token.toString(t.type));
9c69615a4876 Added method insertEmptyTokenBefore() to Lexer. Made some fixes, too.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 413
diff changeset
2474 assert(text.ptr <= t.end && t.end <= end, Token.toString(t.type));
9c69615a4876 Added method insertEmptyTokenBefore() to Lexer. Made some fixes, too.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 413
diff changeset
2475
9c69615a4876 Added method insertEmptyTokenBefore() to Lexer. Made some fixes, too.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 413
diff changeset
2476 auto prev_t = t.prev;
9c69615a4876 Added method insertEmptyTokenBefore() to Lexer. Made some fixes, too.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 413
diff changeset
2477 auto new_t = new Token;
9c69615a4876 Added method insertEmptyTokenBefore() to Lexer. Made some fixes, too.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 413
diff changeset
2478 new_t.type = TOK.Empty;
9c69615a4876 Added method insertEmptyTokenBefore() to Lexer. Made some fixes, too.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 413
diff changeset
2479 new_t.start = new_t.end = prev_t.end;
9c69615a4876 Added method insertEmptyTokenBefore() to Lexer. Made some fixes, too.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 413
diff changeset
2480 // Link in new token.
9c69615a4876 Added method insertEmptyTokenBefore() to Lexer. Made some fixes, too.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 413
diff changeset
2481 prev_t.next = new_t;
9c69615a4876 Added method insertEmptyTokenBefore() to Lexer. Made some fixes, too.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 413
diff changeset
2482 new_t.prev = prev_t;
9c69615a4876 Added method insertEmptyTokenBefore() to Lexer. Made some fixes, too.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 413
diff changeset
2483 new_t.next = t;
9c69615a4876 Added method insertEmptyTokenBefore() to Lexer. Made some fixes, too.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 413
diff changeset
2484 t.prev = new_t;
9c69615a4876 Added method insertEmptyTokenBefore() to Lexer. Made some fixes, too.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 413
diff changeset
2485 return new_t;
9c69615a4876 Added method insertEmptyTokenBefore() to Lexer. Made some fixes, too.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 413
diff changeset
2486 }
105
df34ec47fb81 - Added getState() method and State struct to Lexer.
aziz
parents: 103
diff changeset
2487
422
ad7977fe315a Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 420
diff changeset
2488 void updateErrorLoc(char* columnPos)
103
511a1aa25896 - Added reportErrors member to Lexer. Moved peek() down and rewrote it a bit making use of reportErrors. error() uses reportErrors too.
aziz
parents: 94
diff changeset
2489 {
422
ad7977fe315a Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 420
diff changeset
2490 updateErrorLoc(this.loc, this.lineBegin, columnPos);
239
7911f6a92e6e - Added 'new' and 'delete' declarations to Token and uncommented next and prev members. Added HEAD to TOK.
aziz
parents: 223
diff changeset
2491 }
7911f6a92e6e - Added 'new' and 'delete' declarations to Token and uncommented next and prev members. Added HEAD to TOK.
aziz
parents: 223
diff changeset
2492
422
ad7977fe315a Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 420
diff changeset
2493 void updateErrorLoc(uint lineNum, char* lineBegin, char* columnPos)
239
7911f6a92e6e - Added 'new' and 'delete' declarations to Token and uncommented next and prev members. Added HEAD to TOK.
aziz
parents: 223
diff changeset
2494 {
422
ad7977fe315a Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 420
diff changeset
2495 errorLoc.set(this.errorLineNum(lineNum), lineBegin, columnPos);
239
7911f6a92e6e - Added 'new' and 'delete' declarations to Token and uncommented next and prev members. Added HEAD to TOK.
aziz
parents: 223
diff changeset
2496 }
7911f6a92e6e - Added 'new' and 'delete' declarations to Token and uncommented next and prev members. Added HEAD to TOK.
aziz
parents: 223
diff changeset
2497
422
ad7977fe315a Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 420
diff changeset
2498 uint errorLineNum(uint loc)
239
7911f6a92e6e - Added 'new' and 'delete' declarations to Token and uncommented next and prev members. Added HEAD to TOK.
aziz
parents: 223
diff changeset
2499 {
422
ad7977fe315a Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 420
diff changeset
2500 // ∆loc + line_num_of(#line)
ad7977fe315a Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 420
diff changeset
2501 return loc - this.loc_old + this.loc_hline;
18
c48d2125f1e2 - Moved code for scanning character literals to separate function.
aziz
parents: 17
diff changeset
2502 }
c48d2125f1e2 - Moved code for scanning character literals to separate function.
aziz
parents: 17
diff changeset
2503
422
ad7977fe315a Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 420
diff changeset
2504 void error(char* columnPos, MID mid, ...)
207
481ed2b63a49 - Added contracts to method scan().
aziz
parents: 131
diff changeset
2505 {
422
ad7977fe315a Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 420
diff changeset
2506 updateErrorLoc(columnPos);
ad7977fe315a Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 420
diff changeset
2507 errors ~= new Information(InfoType.Lexer, mid, errorLoc.clone, Format(_arguments, _argptr, GetMsg(mid)));
ad7977fe315a Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 420
diff changeset
2508 }
207
481ed2b63a49 - Added contracts to method scan().
aziz
parents: 131
diff changeset
2509
422
ad7977fe315a Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 420
diff changeset
2510 void error(uint lineNum, char* lineBegin, char* columnPos, MID mid, ...)
ad7977fe315a Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 420
diff changeset
2511 {
ad7977fe315a Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 420
diff changeset
2512 updateErrorLoc(lineNum, lineBegin, columnPos);
ad7977fe315a Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 420
diff changeset
2513 errors ~= new Information(InfoType.Lexer, mid, errorLoc.clone, Format(_arguments, _argptr, GetMsg(mid)));
207
481ed2b63a49 - Added contracts to method scan().
aziz
parents: 131
diff changeset
2514 }
481ed2b63a49 - Added contracts to method scan().
aziz
parents: 131
diff changeset
2515
306
9c866aadcb5b - Moved code out of main() to separate functions.
aziz
parents: 272
diff changeset
2516 Token* getTokens()
4
92df59b1ec4a - Started implementation of scan().
aziz
parents: 3
diff changeset
2517 {
306
9c866aadcb5b - Moved code out of main() to separate functions.
aziz
parents: 272
diff changeset
2518 while (nextToken() != TOK.EOF)
9c866aadcb5b - Moved code out of main() to separate functions.
aziz
parents: 272
diff changeset
2519 {}
9c866aadcb5b - Moved code out of main() to separate functions.
aziz
parents: 272
diff changeset
2520 return head;
3
4bbce78bfb1e - Added TOK enum.
aziz
parents: 2
diff changeset
2521 }
48
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
2522
422
ad7977fe315a Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 420
diff changeset
2523 static void loadKeywords(ref Identifier[string] table)
ad7977fe315a Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 420
diff changeset
2524 {
ad7977fe315a Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 420
diff changeset
2525 foreach(k; keywords)
ad7977fe315a Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 420
diff changeset
2526 table[k.str] = k;
ad7977fe315a Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 420
diff changeset
2527 }
ad7977fe315a Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 420
diff changeset
2528
365
ed67acc82268 - Added option includes to config.d.
aziz
parents: 360
diff changeset
2529 static bool isNonReservedIdentifier(char[] ident)
ed67acc82268 - Added option includes to config.d.
aziz
parents: 360
diff changeset
2530 {
382
66477017cb95 Applied fixes to isNonReservedIdentifier().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 371
diff changeset
2531 if (ident.length == 0)
66477017cb95 Applied fixes to isNonReservedIdentifier().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 371
diff changeset
2532 return false;
66477017cb95 Applied fixes to isNonReservedIdentifier().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 371
diff changeset
2533
365
ed67acc82268 - Added option includes to config.d.
aziz
parents: 360
diff changeset
2534 static Identifier[string] reserved_ids_table;
ed67acc82268 - Added option includes to config.d.
aziz
parents: 360
diff changeset
2535 if (reserved_ids_table is null)
422
ad7977fe315a Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 420
diff changeset
2536 loadKeywords(reserved_ids_table);
365
ed67acc82268 - Added option includes to config.d.
aziz
parents: 360
diff changeset
2537
382
66477017cb95 Applied fixes to isNonReservedIdentifier().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 371
diff changeset
2538 size_t idx = 1; // Index to the 2nd character in ident.
66477017cb95 Applied fixes to isNonReservedIdentifier().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 371
diff changeset
2539 dchar isFirstCharUniAlpha()
66477017cb95 Applied fixes to isNonReservedIdentifier().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 371
diff changeset
2540 {
66477017cb95 Applied fixes to isNonReservedIdentifier().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 371
diff changeset
2541 idx = 0;
66477017cb95 Applied fixes to isNonReservedIdentifier().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 371
diff changeset
2542 // NB: decode() could throw an Exception which would be
66477017cb95 Applied fixes to isNonReservedIdentifier().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 371
diff changeset
2543 // caught by the next try-catch-block.
66477017cb95 Applied fixes to isNonReservedIdentifier().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 371
diff changeset
2544 return isUniAlpha(std.utf.decode(ident, idx));
66477017cb95 Applied fixes to isNonReservedIdentifier().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 371
diff changeset
2545 }
66477017cb95 Applied fixes to isNonReservedIdentifier().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 371
diff changeset
2546
365
ed67acc82268 - Added option includes to config.d.
aziz
parents: 360
diff changeset
2547 try
382
66477017cb95 Applied fixes to isNonReservedIdentifier().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 371
diff changeset
2548 {
66477017cb95 Applied fixes to isNonReservedIdentifier().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 371
diff changeset
2549 if (isidbeg(ident[0]) ||
66477017cb95 Applied fixes to isNonReservedIdentifier().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 371
diff changeset
2550 ident[0] & 128 && isFirstCharUniAlpha())
66477017cb95 Applied fixes to isNonReservedIdentifier().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 371
diff changeset
2551 {
66477017cb95 Applied fixes to isNonReservedIdentifier().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 371
diff changeset
2552 foreach (dchar c; ident[idx..$])
66477017cb95 Applied fixes to isNonReservedIdentifier().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 371
diff changeset
2553 if (!isident(c) && !isUniAlpha(c))
66477017cb95 Applied fixes to isNonReservedIdentifier().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 371
diff changeset
2554 return false;
66477017cb95 Applied fixes to isNonReservedIdentifier().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 371
diff changeset
2555 }
66477017cb95 Applied fixes to isNonReservedIdentifier().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 371
diff changeset
2556 }
66477017cb95 Applied fixes to isNonReservedIdentifier().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 371
diff changeset
2557 catch (Exception)
365
ed67acc82268 - Added option includes to config.d.
aziz
parents: 360
diff changeset
2558 return false;
ed67acc82268 - Added option includes to config.d.
aziz
parents: 360
diff changeset
2559
ed67acc82268 - Added option includes to config.d.
aziz
parents: 360
diff changeset
2560 return !(ident in reserved_ids_table);
ed67acc82268 - Added option includes to config.d.
aziz
parents: 360
diff changeset
2561 }
ed67acc82268 - Added option includes to config.d.
aziz
parents: 360
diff changeset
2562
430
e6c759e151cd Fixed a few things regarding encoding/decoding UTF-8 sequences.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 426
diff changeset
2563 /++
424
bb3cb00feeb2 Applied some fixes to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 422
diff changeset
2564 Returns true if d can be encoded as a UTF-8 sequence.
bb3cb00feeb2 Applied some fixes to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 422
diff changeset
2565 +/
bb3cb00feeb2 Applied some fixes to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 422
diff changeset
2566 bool isEncodable(dchar d)
bb3cb00feeb2 Applied some fixes to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 422
diff changeset
2567 {
bb3cb00feeb2 Applied some fixes to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 422
diff changeset
2568 return d < 0xD800 ||
430
e6c759e151cd Fixed a few things regarding encoding/decoding UTF-8 sequences.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 426
diff changeset
2569 (d > 0xDFFF && d <= 0x10FFFF);
e6c759e151cd Fixed a few things regarding encoding/decoding UTF-8 sequences.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 426
diff changeset
2570 }
e6c759e151cd Fixed a few things regarding encoding/decoding UTF-8 sequences.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 426
diff changeset
2571
e6c759e151cd Fixed a few things regarding encoding/decoding UTF-8 sequences.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 426
diff changeset
2572 /++
e6c759e151cd Fixed a few things regarding encoding/decoding UTF-8 sequences.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 426
diff changeset
2573 There are a total of 66 noncharacters.
e6c759e151cd Fixed a few things regarding encoding/decoding UTF-8 sequences.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 426
diff changeset
2574 Returns true if this is one of them.
e6c759e151cd Fixed a few things regarding encoding/decoding UTF-8 sequences.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 426
diff changeset
2575 See_also: Chapter 16.7 Noncharacters in Unicode 5.0
e6c759e151cd Fixed a few things regarding encoding/decoding UTF-8 sequences.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 426
diff changeset
2576 +/
e6c759e151cd Fixed a few things regarding encoding/decoding UTF-8 sequences.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 426
diff changeset
2577 bool isNoncharacter(dchar d)
e6c759e151cd Fixed a few things regarding encoding/decoding UTF-8 sequences.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 426
diff changeset
2578 {
e6c759e151cd Fixed a few things regarding encoding/decoding UTF-8 sequences.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 426
diff changeset
2579 return 0xFDD0 <= d && d <= 0xFDEF || // 32
e6c759e151cd Fixed a few things regarding encoding/decoding UTF-8 sequences.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 426
diff changeset
2580 d <= 0x10FFFF && (d & 0xFFFF) >= 0xFFFE; // 34
e6c759e151cd Fixed a few things regarding encoding/decoding UTF-8 sequences.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 426
diff changeset
2581 }
e6c759e151cd Fixed a few things regarding encoding/decoding UTF-8 sequences.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 426
diff changeset
2582
e6c759e151cd Fixed a few things regarding encoding/decoding UTF-8 sequences.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 426
diff changeset
2583 /++
e6c759e151cd Fixed a few things regarding encoding/decoding UTF-8 sequences.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 426
diff changeset
2584 Returns true if this character is not a noncharacter, not a surrogate
e6c759e151cd Fixed a few things regarding encoding/decoding UTF-8 sequences.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 426
diff changeset
2585 code point and not higher than 0x10FFFF.
e6c759e151cd Fixed a few things regarding encoding/decoding UTF-8 sequences.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 426
diff changeset
2586 +/
e6c759e151cd Fixed a few things regarding encoding/decoding UTF-8 sequences.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 426
diff changeset
2587 bool isValidDecodedChar(dchar d)
e6c759e151cd Fixed a few things regarding encoding/decoding UTF-8 sequences.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 426
diff changeset
2588 {
e6c759e151cd Fixed a few things regarding encoding/decoding UTF-8 sequences.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 426
diff changeset
2589 return d < 0xD800 ||
e6c759e151cd Fixed a few things regarding encoding/decoding UTF-8 sequences.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 426
diff changeset
2590 (d > 0xDFFF && d < 0xFDD0) ||
e6c759e151cd Fixed a few things regarding encoding/decoding UTF-8 sequences.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 426
diff changeset
2591 (d > 0xFDEF && d <= 0x10FFFF && (d & 0xFFFF) < 0xFFFE);
424
bb3cb00feeb2 Applied some fixes to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 422
diff changeset
2592 }
bb3cb00feeb2 Applied some fixes to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 422
diff changeset
2593
426
3f7790d3f9d6 Improved decodeUTF8().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 425
diff changeset
2594 /// Is this a trail byte of a UTF-8 sequence?
425
6bf936bf3356 Added own UTF-8 decoding function.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 424
diff changeset
2595 bool isTrailByte(ubyte b)
6bf936bf3356 Added own UTF-8 decoding function.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 424
diff changeset
2596 {
426
3f7790d3f9d6 Improved decodeUTF8().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 425
diff changeset
2597 return (b & 0xC0) == 0x80; // 10xx_xxxx
425
6bf936bf3356 Added own UTF-8 decoding function.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 424
diff changeset
2598 }
6bf936bf3356 Added own UTF-8 decoding function.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 424
diff changeset
2599
426
3f7790d3f9d6 Improved decodeUTF8().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 425
diff changeset
2600 /// Is this a lead byte of a UTF-8 sequence?
3f7790d3f9d6 Improved decodeUTF8().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 425
diff changeset
2601 bool isLeadByte(ubyte b)
424
bb3cb00feeb2 Applied some fixes to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 422
diff changeset
2602 {
426
3f7790d3f9d6 Improved decodeUTF8().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 425
diff changeset
2603 return (b & 0xC0) == 0xC0; // 11xx_xxxx
424
bb3cb00feeb2 Applied some fixes to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 422
diff changeset
2604 }
425
6bf936bf3356 Added own UTF-8 decoding function.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 424
diff changeset
2605
6bf936bf3356 Added own UTF-8 decoding function.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 424
diff changeset
2606 dchar decodeUTF8()
6bf936bf3356 Added own UTF-8 decoding function.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 424
diff changeset
2607 {
6bf936bf3356 Added own UTF-8 decoding function.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 424
diff changeset
2608 assert(!isascii(*p), "check for ASCII char before calling decodeUTF8().");
6bf936bf3356 Added own UTF-8 decoding function.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 424
diff changeset
2609 char* p = this.p;
6bf936bf3356 Added own UTF-8 decoding function.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 424
diff changeset
2610 dchar d = *p;
6bf936bf3356 Added own UTF-8 decoding function.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 424
diff changeset
2611
426
3f7790d3f9d6 Improved decodeUTF8().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 425
diff changeset
2612 ++p; // Move to second byte.
3f7790d3f9d6 Improved decodeUTF8().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 425
diff changeset
2613 // Error if second byte is not a trail byte.
3f7790d3f9d6 Improved decodeUTF8().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 425
diff changeset
2614 if (!isTrailByte(*p))
3f7790d3f9d6 Improved decodeUTF8().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 425
diff changeset
2615 goto Lerr2;
3f7790d3f9d6 Improved decodeUTF8().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 425
diff changeset
2616
425
6bf936bf3356 Added own UTF-8 decoding function.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 424
diff changeset
2617 // Check for overlong sequences.
6bf936bf3356 Added own UTF-8 decoding function.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 424
diff changeset
2618 switch (d)
6bf936bf3356 Added own UTF-8 decoding function.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 424
diff changeset
2619 {
426
3f7790d3f9d6 Improved decodeUTF8().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 425
diff changeset
2620 case 0xE0, // 11100000 100xxxxx
3f7790d3f9d6 Improved decodeUTF8().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 425
diff changeset
2621 0xF0, // 11110000 1000xxxx
3f7790d3f9d6 Improved decodeUTF8().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 425
diff changeset
2622 0xF8, // 11111000 10000xxx
3f7790d3f9d6 Improved decodeUTF8().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 425
diff changeset
2623 0xFC: // 11111100 100000xx
3f7790d3f9d6 Improved decodeUTF8().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 425
diff changeset
2624 if ((*p & d) == 0x80)
425
6bf936bf3356 Added own UTF-8 decoding function.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 424
diff changeset
2625 goto Lerr;
6bf936bf3356 Added own UTF-8 decoding function.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 424
diff changeset
2626 default:
6bf936bf3356 Added own UTF-8 decoding function.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 424
diff changeset
2627 if ((d & 0xFE) == 0xC0) // 1100000x
6bf936bf3356 Added own UTF-8 decoding function.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 424
diff changeset
2628 goto Lerr;
6bf936bf3356 Added own UTF-8 decoding function.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 424
diff changeset
2629 }
6bf936bf3356 Added own UTF-8 decoding function.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 424
diff changeset
2630
426
3f7790d3f9d6 Improved decodeUTF8().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 425
diff changeset
2631 const char[] checkNextByte = "if (!isTrailByte(*++p))"
3f7790d3f9d6 Improved decodeUTF8().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 425
diff changeset
2632 " goto Lerr2;";
3f7790d3f9d6 Improved decodeUTF8().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 425
diff changeset
2633 const char[] appendSixBits = "d = (d << 6) | *p & 0b0011_1111;";
3f7790d3f9d6 Improved decodeUTF8().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 425
diff changeset
2634
425
6bf936bf3356 Added own UTF-8 decoding function.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 424
diff changeset
2635 // Decode
6bf936bf3356 Added own UTF-8 decoding function.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 424
diff changeset
2636 if ((d & 0b1110_0000) == 0b1100_0000)
6bf936bf3356 Added own UTF-8 decoding function.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 424
diff changeset
2637 {
6bf936bf3356 Added own UTF-8 decoding function.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 424
diff changeset
2638 // 110xxxxx 10xxxxxx
6bf936bf3356 Added own UTF-8 decoding function.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 424
diff changeset
2639 d &= 0b0001_1111;
426
3f7790d3f9d6 Improved decodeUTF8().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 425
diff changeset
2640 mixin(appendSixBits);
425
6bf936bf3356 Added own UTF-8 decoding function.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 424
diff changeset
2641 }
6bf936bf3356 Added own UTF-8 decoding function.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 424
diff changeset
2642 else if ((d & 0b1111_0000) == 0b1110_0000)
6bf936bf3356 Added own UTF-8 decoding function.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 424
diff changeset
2643 {
6bf936bf3356 Added own UTF-8 decoding function.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 424
diff changeset
2644 // 1110xxxx 10xxxxxx 10xxxxxx
6bf936bf3356 Added own UTF-8 decoding function.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 424
diff changeset
2645 d &= 0b0000_1111;
426
3f7790d3f9d6 Improved decodeUTF8().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 425
diff changeset
2646 mixin(appendSixBits ~
3f7790d3f9d6 Improved decodeUTF8().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 425
diff changeset
2647 checkNextByte ~ appendSixBits);
425
6bf936bf3356 Added own UTF-8 decoding function.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 424
diff changeset
2648 }
6bf936bf3356 Added own UTF-8 decoding function.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 424
diff changeset
2649 else if ((d & 0b1111_1000) == 0b1111_0000)
6bf936bf3356 Added own UTF-8 decoding function.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 424
diff changeset
2650 {
6bf936bf3356 Added own UTF-8 decoding function.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 424
diff changeset
2651 // 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx
6bf936bf3356 Added own UTF-8 decoding function.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 424
diff changeset
2652 d &= 0b0000_0111;
426
3f7790d3f9d6 Improved decodeUTF8().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 425
diff changeset
2653 mixin(appendSixBits ~
3f7790d3f9d6 Improved decodeUTF8().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 425
diff changeset
2654 checkNextByte ~ appendSixBits ~
3f7790d3f9d6 Improved decodeUTF8().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 425
diff changeset
2655 checkNextByte ~ appendSixBits);
425
6bf936bf3356 Added own UTF-8 decoding function.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 424
diff changeset
2656 }
6bf936bf3356 Added own UTF-8 decoding function.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 424
diff changeset
2657 else
6bf936bf3356 Added own UTF-8 decoding function.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 424
diff changeset
2658 // 5 and 6 byte UTF-8 sequences are not allowed yet.
6bf936bf3356 Added own UTF-8 decoding function.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 424
diff changeset
2659 // 111110xx 10xxxxxx 10xxxxxx 10xxxxxx 10xxxxxx
6bf936bf3356 Added own UTF-8 decoding function.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 424
diff changeset
2660 // 1111110x 10xxxxxx 10xxxxxx 10xxxxxx 10xxxxxx 10xxxxxx
6bf936bf3356 Added own UTF-8 decoding function.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 424
diff changeset
2661 goto Lerr;
6bf936bf3356 Added own UTF-8 decoding function.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 424
diff changeset
2662
426
3f7790d3f9d6 Improved decodeUTF8().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 425
diff changeset
2663 assert(isTrailByte(*p));
3f7790d3f9d6 Improved decodeUTF8().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 425
diff changeset
2664
425
6bf936bf3356 Added own UTF-8 decoding function.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 424
diff changeset
2665 if (!isEncodable(d))
6bf936bf3356 Added own UTF-8 decoding function.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 424
diff changeset
2666 {
6bf936bf3356 Added own UTF-8 decoding function.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 424
diff changeset
2667 Lerr:
426
3f7790d3f9d6 Improved decodeUTF8().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 425
diff changeset
2668 // Three cases:
3f7790d3f9d6 Improved decodeUTF8().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 425
diff changeset
2669 // *) the UTF-8 sequence was successfully decoded but the resulting
3f7790d3f9d6 Improved decodeUTF8().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 425
diff changeset
2670 // character is invalid.
3f7790d3f9d6 Improved decodeUTF8().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 425
diff changeset
2671 // p points to last trail byte in the sequence.
3f7790d3f9d6 Improved decodeUTF8().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 425
diff changeset
2672 // *) the UTF-8 sequence is overlong.
3f7790d3f9d6 Improved decodeUTF8().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 425
diff changeset
2673 // p points to second byte in the sequence.
3f7790d3f9d6 Improved decodeUTF8().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 425
diff changeset
2674 // *) the UTF-8 sequence has more than 4 bytes or starts with
3f7790d3f9d6 Improved decodeUTF8().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 425
diff changeset
2675 // a trail byte.
3f7790d3f9d6 Improved decodeUTF8().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 425
diff changeset
2676 // p points to second byte in the sequence.
3f7790d3f9d6 Improved decodeUTF8().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 425
diff changeset
2677 assert(isTrailByte(*p));
3f7790d3f9d6 Improved decodeUTF8().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 425
diff changeset
2678 // Move to next ASCII character or lead byte of a UTF-8 sequence.
3f7790d3f9d6 Improved decodeUTF8().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 425
diff changeset
2679 while (p < (end-1) && isTrailByte(*p))
3f7790d3f9d6 Improved decodeUTF8().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 425
diff changeset
2680 ++p;
425
6bf936bf3356 Added own UTF-8 decoding function.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 424
diff changeset
2681 --p;
426
3f7790d3f9d6 Improved decodeUTF8().
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 425
diff changeset
2682 assert(!isTrailByte(p[1]));
425
6bf936bf3356 Added own UTF-8 decoding function.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 424
diff changeset
2683 Lerr2:
430
e6c759e151cd Fixed a few things regarding encoding/decoding UTF-8 sequences.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 426
diff changeset
2684 d = REPLACEMENT_CHAR;
425
6bf936bf3356 Added own UTF-8 decoding function.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 424
diff changeset
2685 error(this.p, MID.InvalidUTF8Sequence);
6bf936bf3356 Added own UTF-8 decoding function.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 424
diff changeset
2686 }
6bf936bf3356 Added own UTF-8 decoding function.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 424
diff changeset
2687
6bf936bf3356 Added own UTF-8 decoding function.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 424
diff changeset
2688 this.p = p;
6bf936bf3356 Added own UTF-8 decoding function.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 424
diff changeset
2689 return d;
6bf936bf3356 Added own UTF-8 decoding function.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 424
diff changeset
2690 }
424
bb3cb00feeb2 Applied some fixes to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 422
diff changeset
2691
bb3cb00feeb2 Applied some fixes to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 422
diff changeset
2692 private void encodeUTF8(ref char[] str, dchar d)
48
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
2693 {
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
2694 char[6] b;
430
e6c759e151cd Fixed a few things regarding encoding/decoding UTF-8 sequences.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 426
diff changeset
2695 assert(!isascii(d), "check for ASCII char before calling encodeUTF8().");
424
bb3cb00feeb2 Applied some fixes to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 422
diff changeset
2696 assert(isEncodable(d), "check that 'd' is encodable before calling encodeUTF8().");
430
e6c759e151cd Fixed a few things regarding encoding/decoding UTF-8 sequences.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 426
diff changeset
2697
48
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
2698 if (d < 0x800)
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
2699 {
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
2700 b[0] = 0xC0 | (d >> 6);
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
2701 b[1] = 0x80 | (d & 0x3F);
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
2702 str ~= b[0..2];
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
2703 }
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
2704 else if (d < 0x10000)
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
2705 {
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
2706 b[0] = 0xE0 | (d >> 12);
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
2707 b[1] = 0x80 | ((d >> 6) & 0x3F);
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
2708 b[2] = 0x80 | (d & 0x3F);
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
2709 str ~= b[0..3];
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
2710 }
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
2711 else if (d < 0x200000)
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
2712 {
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
2713 b[0] = 0xF0 | (d >> 18);
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
2714 b[1] = 0x80 | ((d >> 12) & 0x3F);
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
2715 b[2] = 0x80 | ((d >> 6) & 0x3F);
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
2716 b[3] = 0x80 | (d & 0x3F);
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
2717 str ~= b[0..4];
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
2718 }
424
bb3cb00feeb2 Applied some fixes to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 422
diff changeset
2719 /+ // There are no 5 and 6 byte UTF-8 sequences yet.
48
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
2720 else if (d < 0x4000000)
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
2721 {
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
2722 b[0] = 0xF8 | (d >> 24);
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
2723 b[1] = 0x80 | ((d >> 18) & 0x3F);
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
2724 b[2] = 0x80 | ((d >> 12) & 0x3F);
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
2725 b[3] = 0x80 | ((d >> 6) & 0x3F);
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
2726 b[4] = 0x80 | (d & 0x3F);
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
2727 str ~= b[0..5];
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
2728 }
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
2729 else if (d < 0x80000000)
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
2730 {
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
2731 b[0] = 0xFC | (d >> 30);
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
2732 b[1] = 0x80 | ((d >> 24) & 0x3F);
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
2733 b[2] = 0x80 | ((d >> 18) & 0x3F);
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
2734 b[3] = 0x80 | ((d >> 12) & 0x3F);
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
2735 b[4] = 0x80 | ((d >> 6) & 0x3F);
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
2736 b[5] = 0x80 | (d & 0x3F);
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
2737 str ~= b[0..6];
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
2738 }
424
bb3cb00feeb2 Applied some fixes to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 422
diff changeset
2739 +/
bb3cb00feeb2 Applied some fixes to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 422
diff changeset
2740 else
bb3cb00feeb2 Applied some fixes to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 422
diff changeset
2741 assert(0);
48
c2e0e0269c28 - Added code for scanning escape string literals.
aziz
parents: 47
diff changeset
2742 }
5
79b4e8848794 - Started writing XML generator.
aziz
parents: 4
diff changeset
2743 }
39
69b940398d7b - Added unittest to test correct parsing of operator tokens.
aziz
parents: 38
diff changeset
2744
69b940398d7b - Added unittest to test correct parsing of operator tokens.
aziz
parents: 38
diff changeset
2745 unittest
69b940398d7b - Added unittest to test correct parsing of operator tokens.
aziz
parents: 38
diff changeset
2746 {
391
33b566df6af4 Migrated project to Tango.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 390
diff changeset
2747 Stdout("Testing Lexer.\n");
413
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
2748 struct Pair
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
2749 {
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
2750 char[] token;
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
2751 TOK type;
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
2752 }
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
2753 static Pair[] pairs = [
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
2754 {"//çay\n", TOK.Comment}, {"&", TOK.AndBinary},
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
2755 {"/*çağ*/", TOK.Comment}, {"&&", TOK.AndLogical},
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
2756 {"/+çak+/", TOK.Comment}, {"&=", TOK.AndAssign},
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
2757 {">", TOK.Greater}, {"+", TOK.Plus},
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
2758 {">=", TOK.GreaterEqual}, {"++", TOK.PlusPlus},
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
2759 {">>", TOK.RShift}, {"+=", TOK.PlusAssign},
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
2760 {">>=", TOK.RShiftAssign}, {"-", TOK.Minus},
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
2761 {">>>", TOK.URShift}, {"--", TOK.MinusMinus},
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
2762 {">>>=", TOK.URShiftAssign}, {"-=", TOK.MinusAssign},
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
2763 {"<", TOK.Less}, {"=", TOK.Assign},
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
2764 {"<=", TOK.LessEqual}, {"==", TOK.Equal},
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
2765 {"<>", TOK.LorG}, {"~", TOK.Tilde},
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
2766 {"<>=", TOK.LorEorG}, {"~=", TOK.CatAssign},
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
2767 {"<<", TOK.LShift}, {"*", TOK.Mul},
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
2768 {"<<=", TOK.LShiftAssign}, {"*=", TOK.MulAssign},
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
2769 {"!", TOK.Not}, {"/", TOK.Div},
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
2770 {"!=", TOK.NotEqual}, {"/=", TOK.DivAssign},
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
2771 {"!<", TOK.UorGorE}, {"^", TOK.Xor},
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
2772 {"!>", TOK.UorLorE}, {"^=", TOK.XorAssign},
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
2773 {"!<=", TOK.UorG}, {"%", TOK.Mod},
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
2774 {"!>=", TOK.UorL}, {"%=", TOK.ModAssign},
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
2775 {"!<>", TOK.UorE}, {"(", TOK.LParen},
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
2776 {"!<>=", TOK.Unordered}, {")", TOK.RParen},
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
2777 {".", TOK.Dot}, {"[", TOK.LBracket},
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
2778 {"..", TOK.Slice}, {"]", TOK.RBracket},
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
2779 {"...", TOK.Ellipses}, {"{", TOK.LBrace},
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
2780 {"|", TOK.OrBinary}, {"}", TOK.RBrace},
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
2781 {"||", TOK.OrLogical}, {":", TOK.Colon},
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
2782 {"|=", TOK.OrAssign}, {";", TOK.Semicolon},
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
2783 {"?", TOK.Question}, {",", TOK.Comma},
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
2784 {"$", TOK.Dollar}, {"cam", TOK.Identifier},
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
2785 {"çay", TOK.Identifier}, {".0", TOK.Float64},
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
2786 {"0", TOK.Int32},
40
9d5ceb0f8be9 - Added more tokens for testing.
aziz
parents: 39
diff changeset
2787 ];
39
69b940398d7b - Added unittest to test correct parsing of operator tokens.
aziz
parents: 38
diff changeset
2788
69b940398d7b - Added unittest to test correct parsing of operator tokens.
aziz
parents: 38
diff changeset
2789 char[] src;
69b940398d7b - Added unittest to test correct parsing of operator tokens.
aziz
parents: 38
diff changeset
2790
413
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
2791 foreach (pair; pairs)
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
2792 src ~= pair.token ~ " ";
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
2793
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
2794 assert(pairs[0].token == "//çay\n");
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
2795 // Remove \n after src has been constructed.
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
2796 // It won't be part of the scanned token string.
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
2797 pairs[0].token = "//çay";
39
69b940398d7b - Added unittest to test correct parsing of operator tokens.
aziz
parents: 38
diff changeset
2798
69b940398d7b - Added unittest to test correct parsing of operator tokens.
aziz
parents: 38
diff changeset
2799 auto lx = new Lexer(src, "");
350
4ea6759300cf - Fixed unittests in module Lexer.
aziz
parents: 346
diff changeset
2800 auto token = lx.getTokens();
39
69b940398d7b - Added unittest to test correct parsing of operator tokens.
aziz
parents: 38
diff changeset
2801
350
4ea6759300cf - Fixed unittests in module Lexer.
aziz
parents: 346
diff changeset
2802 uint i;
4ea6759300cf - Fixed unittests in module Lexer.
aziz
parents: 346
diff changeset
2803 assert(token == lx.head);
4ea6759300cf - Fixed unittests in module Lexer.
aziz
parents: 346
diff changeset
2804 token = token.next;
4ea6759300cf - Fixed unittests in module Lexer.
aziz
parents: 346
diff changeset
2805 do
4ea6759300cf - Fixed unittests in module Lexer.
aziz
parents: 346
diff changeset
2806 {
413
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
2807 assert(i < pairs.length);
0fd78fdcb982 Added an alternative scan() method to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 412
diff changeset
2808 assert(token.srcText == pairs[i].token, Format("Scanned '{0}' but expected '{1}'", token.srcText, pairs[i].token));
350
4ea6759300cf - Fixed unittests in module Lexer.
aziz
parents: 346
diff changeset
2809 ++i;
4ea6759300cf - Fixed unittests in module Lexer.
aziz
parents: 346
diff changeset
2810 token = token.next;
4ea6759300cf - Fixed unittests in module Lexer.
aziz
parents: 346
diff changeset
2811 } while (token.type != TOK.EOF)
41
2b7be1d67d4d - Optimized scanner of block comments.
aziz
parents: 40
diff changeset
2812 }
55
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
2813
67
996065105910 - Fix: variadic arguments are local to the variadic function. Parameters are converted to an array of strings first and then passed to the constructor of class Problem.
aziz
parents: 66
diff changeset
2814 unittest
996065105910 - Fix: variadic arguments are local to the variadic function. Parameters are converted to an array of strings first and then passed to the constructor of class Problem.
aziz
parents: 66
diff changeset
2815 {
422
ad7977fe315a Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 420
diff changeset
2816 Stdout("Testing method Lexer.peek()\n");
ad7977fe315a Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 420
diff changeset
2817 string sourceText = "unittest { }";
ad7977fe315a Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 420
diff changeset
2818 auto lx = new Lexer(sourceText, null);
ad7977fe315a Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 420
diff changeset
2819
ad7977fe315a Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 420
diff changeset
2820 Token* next = lx.head;
ad7977fe315a Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 420
diff changeset
2821 lx.peek(next);
ad7977fe315a Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 420
diff changeset
2822 assert(next.type == TOK.Unittest);
ad7977fe315a Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 420
diff changeset
2823 lx.peek(next);
ad7977fe315a Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 420
diff changeset
2824 assert(next.type == TOK.LBrace);
ad7977fe315a Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 420
diff changeset
2825 lx.peek(next);
ad7977fe315a Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 420
diff changeset
2826 assert(next.type == TOK.RBrace);
ad7977fe315a Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 420
diff changeset
2827 lx.peek(next);
ad7977fe315a Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 420
diff changeset
2828 assert(next.type == TOK.EOF);
ad7977fe315a Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 420
diff changeset
2829 }
ad7977fe315a Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 420
diff changeset
2830
ad7977fe315a Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 420
diff changeset
2831 unittest
ad7977fe315a Added support for column numbers in error messages.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 420
diff changeset
2832 {
67
996065105910 - Fix: variadic arguments are local to the variadic function. Parameters are converted to an array of strings first and then passed to the constructor of class Problem.
aziz
parents: 66
diff changeset
2833 // Numbers unittest
68
7eb83dd38901 - Simplified suffix rule and added a few more numbers to unittest.
aziz
parents: 67
diff changeset
2834 // 0L 0ULi 0_L 0_UL 0x0U 0x0p2 0_Fi 0_e2 0_F 0_i
7eb83dd38901 - Simplified suffix rule and added a few more numbers to unittest.
aziz
parents: 67
diff changeset
2835 // 0u 0U 0uL 0UL 0L 0LU 0Lu
7eb83dd38901 - Simplified suffix rule and added a few more numbers to unittest.
aziz
parents: 67
diff changeset
2836 // 0Li 0f 0F 0fi 0Fi 0i
7eb83dd38901 - Simplified suffix rule and added a few more numbers to unittest.
aziz
parents: 67
diff changeset
2837 // 0b_1_LU 0b1000u
7eb83dd38901 - Simplified suffix rule and added a few more numbers to unittest.
aziz
parents: 67
diff changeset
2838 // 0x232Lu
67
996065105910 - Fix: variadic arguments are local to the variadic function. Parameters are converted to an array of strings first and then passed to the constructor of class Problem.
aziz
parents: 66
diff changeset
2839 }
996065105910 - Fix: variadic arguments are local to the variadic function. Parameters are converted to an array of strings first and then passed to the constructor of class Problem.
aziz
parents: 66
diff changeset
2840
55
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
2841 /// ASCII character properties table.
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
2842 static const int ptable[256] = [
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
2843 0, 0, 0, 0, 0, 0, 0, 0, 0,32, 0,32,32, 0, 0, 0,
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
2844 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
2845 32, 0, 0x2200, 0, 0, 0, 0, 0x2700, 0, 0, 0, 0, 0, 0, 0, 0,
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
2846 7, 7, 7, 7, 7, 7, 7, 7, 6, 6, 0, 0, 0, 0, 0, 0x3f00,
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
2847 0,12,12,12,12,12,12, 8, 8, 8, 8, 8, 8, 8, 8, 8,
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
2848 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 0, 0x5c00, 0, 0,16,
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
2849 0, 0x70c, 0x80c,12,12,12, 0xc0c, 8, 8, 8, 8, 8, 8, 8, 0xa08, 8,
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
2850 8, 8, 0xd08, 8, 0x908, 8, 0xb08, 8, 8, 8, 8, 0, 0, 0, 0, 0,
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
2851 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
2852 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
2853 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
2854 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
2855 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
2856 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
2857 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
2858 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
2859 ];
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
2860
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
2861 enum CProperty
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
2862 {
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
2863 Octal = 1,
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
2864 Digit = 1<<1,
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
2865 Hex = 1<<2,
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
2866 Alpha = 1<<3,
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
2867 Underscore = 1<<4,
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
2868 Whitespace = 1<<5
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
2869 }
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
2870
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
2871 const uint EVMask = 0xFF00; // Bit mask for escape value
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
2872
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
2873 private alias CProperty CP;
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
2874 int isoctal(char c) { return ptable[c] & CP.Octal; }
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
2875 int isdigit(char c) { return ptable[c] & CP.Digit; }
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
2876 int ishexad(char c) { return ptable[c] & CP.Hex; }
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
2877 int isalpha(char c) { return ptable[c] & CP.Alpha; }
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
2878 int isalnum(char c) { return ptable[c] & (CP.Alpha | CP.Digit); }
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
2879 int isidbeg(char c) { return ptable[c] & (CP.Alpha | CP.Underscore); }
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
2880 int isident(char c) { return ptable[c] & (CP.Alpha | CP.Underscore | CP.Digit); }
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
2881 int isspace(char c) { return ptable[c] & CP.Whitespace; }
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
2882 int char2ev(char c) { return ptable[c] >> 8; /*(ptable[c] & EVMask) >> 8;*/ }
424
bb3cb00feeb2 Applied some fixes to class Lexer.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 422
diff changeset
2883 int isascii(uint c) { return c < 128; }
55
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
2884
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
2885 version(gen_ptable)
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
2886 static this()
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
2887 {
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
2888 alias ptable p;
402
22d65b2bef4f Migrated the code that generates the ptable to Tango.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 397
diff changeset
2889 assert(p.length == 256);
55
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
2890 // Initialize character properties table.
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
2891 for (int i; i < p.length; ++i)
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
2892 {
402
22d65b2bef4f Migrated the code that generates the ptable to Tango.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 397
diff changeset
2893 p[i] = 0; // Reset
55
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
2894 if ('0' <= i && i <= '7')
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
2895 p[i] |= CP.Octal;
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
2896 if ('0' <= i && i <= '9')
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
2897 p[i] |= CP.Digit;
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
2898 if (isdigit(i) || 'a' <= i && i <= 'f' || 'A' <= i && i <= 'F')
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
2899 p[i] |= CP.Hex;
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
2900 if ('a' <= i && i <= 'z' || 'A' <= i && i <= 'Z')
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
2901 p[i] |= CP.Alpha;
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
2902 if (i == '_')
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
2903 p[i] |= CP.Underscore;
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
2904 if (i == ' ' || i == '\t' || i == '\v' || i == '\f')
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
2905 p[i] |= CP.Whitespace;
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
2906 }
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
2907 // Store escape sequence values in second byte.
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
2908 assert(CProperty.max <= ubyte.max, "character property flags and escape value byte overlap.");
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
2909 p['\''] |= 39 << 8;
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
2910 p['"'] |= 34 << 8;
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
2911 p['?'] |= 63 << 8;
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
2912 p['\\'] |= 92 << 8;
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
2913 p['a'] |= 7 << 8;
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
2914 p['b'] |= 8 << 8;
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
2915 p['f'] |= 12 << 8;
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
2916 p['n'] |= 10 << 8;
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
2917 p['r'] |= 13 << 8;
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
2918 p['t'] |= 9 << 8;
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
2919 p['v'] |= 11 << 8;
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
2920 // Print a formatted array literal.
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
2921 char[] array = "[\n";
402
22d65b2bef4f Migrated the code that generates the ptable to Tango.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 397
diff changeset
2922 foreach (i, c; ptable)
55
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
2923 {
402
22d65b2bef4f Migrated the code that generates the ptable to Tango.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 397
diff changeset
2924 array ~= Format((c>255?" 0x{0:x},":"{0,2},"), c) ~ (((i+1) % 16) ? "":"\n");
55
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
2925 }
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
2926 array[$-2..$] = "\n]";
402
22d65b2bef4f Migrated the code that generates the ptable to Tango.
Aziz K?ksal <aziz.koeksal@gmail.com>
parents: 397
diff changeset
2927 Stdout(array).newline;
55
5887751f8e04 - Relocated ptable to the bottom of the source file.
aziz
parents: 54
diff changeset
2928 }