Mercurial > projects > dil

annotate trunk/src/Token.d @ 58:50bb7fc9db44

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
- The types of integers are recognized now. - Fix: continue statement missing in integer scanner.
author aziz
date Fri, 29 Jun 2007 05:47:00 +0000
parents 63af7ddf52e1
children 96af5653acef
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
0
8ba2570de175 Initial import.
aziz
parents:
diff changeset
1 /++
8ba2570de175 Initial import.
aziz
parents:
diff changeset
2 Author: Aziz Köksal
8ba2570de175 Initial import.
aziz
parents:
diff changeset
3 License: GPL2
8ba2570de175 Initial import.
aziz
parents:
diff changeset
4 +/
8ba2570de175 Initial import.
aziz
parents:
diff changeset
5 module Token;
8ba2570de175 Initial import.
aziz
parents:
diff changeset
6
8ba2570de175 Initial import.
aziz
parents:
diff changeset
7 struct Position
8ba2570de175 Initial import.
aziz
parents:
diff changeset
8 {
8ba2570de175 Initial import.
aziz
parents:
diff changeset
9 size_t loc;
8ba2570de175 Initial import.
aziz
parents:
diff changeset
10 size_t col;
8ba2570de175 Initial import.
aziz
parents:
diff changeset
11 }
8ba2570de175 Initial import.
aziz
parents:
diff changeset
12
3
4bbce78bfb1e - Added TOK enum.
aziz
parents: 0
diff changeset
13 enum TOK
4bbce78bfb1e - Added TOK enum.
aziz
parents: 0
diff changeset
14 {
4bbce78bfb1e - Added TOK enum.
aziz
parents: 0
diff changeset
15 Identifier,
4
92df59b1ec4a - Started implementation of scan().
aziz
parents: 3
diff changeset
16 Comment,
9
5d6968cc751e - Parsing string and character literals now (rudimentary implementation.)
aziz
parents: 4
diff changeset
17 String,
5d6968cc751e - Parsing string and character literals now (rudimentary implementation.)
aziz
parents: 4
diff changeset
18 Character,
58
50bb7fc9db44 - The types of integers are recognized now.
aziz
parents: 56
diff changeset
19
50bb7fc9db44 - The types of integers are recognized now.
aziz
parents: 56
diff changeset
20 // Numbers
15
c70c028e47dd - Started implementation of lexing numbers.
aziz
parents: 14
diff changeset
21 Number,
58
50bb7fc9db44 - The types of integers are recognized now.
aziz
parents: 56
diff changeset
22 Int32, Int64, Uint32, Uint64,
50bb7fc9db44 - The types of integers are recognized now.
aziz
parents: 56
diff changeset
23
36
3c7210a722f7 - Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents: 35
diff changeset
24
3c7210a722f7 - Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents: 35
diff changeset
25 // Brackets
20
d6adfbd7c513 - Added code for parsing braces.
aziz
parents: 18
diff changeset
26 LParen,
d6adfbd7c513 - Added code for parsing braces.
aziz
parents: 18
diff changeset
27 RParen,
d6adfbd7c513 - Added code for parsing braces.
aziz
parents: 18
diff changeset
28 LBracket,
d6adfbd7c513 - Added code for parsing braces.
aziz
parents: 18
diff changeset
29 RBracket,
d6adfbd7c513 - Added code for parsing braces.
aziz
parents: 18
diff changeset
30 LBrace,
d6adfbd7c513 - Added code for parsing braces.
aziz
parents: 18
diff changeset
31 RBrace,
21
c785c122e4e6 - Added code for parsing Colon, Semicolon, Question, Comma and Dollar.
aziz
parents: 20
diff changeset
32
22
b05fff8e2ce4 - Added code for parsing Dot, Slice and Ellipses tokens.
aziz
parents: 21
diff changeset
33 Dot, Slice, Ellipses,
b05fff8e2ce4 - Added code for parsing Dot, Slice and Ellipses tokens.
aziz
parents: 21
diff changeset
34
36
3c7210a722f7 - Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents: 35
diff changeset
35 // Floating point operators
35
c470b9356e35 - Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents: 32
diff changeset
36 Unordered,
c470b9356e35 - Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents: 32
diff changeset
37 UorE,
c470b9356e35 - Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents: 32
diff changeset
38 UorG,
c470b9356e35 - Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents: 32
diff changeset
39 UorGorE,
c470b9356e35 - Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents: 32
diff changeset
40 UorL,
c470b9356e35 - Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents: 32
diff changeset
41 UorLorE,
36
3c7210a722f7 - Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents: 35
diff changeset
42 LorEorG,
3c7210a722f7 - Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents: 35
diff changeset
43 LorG,
35
c470b9356e35 - Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents: 32
diff changeset
44
36
3c7210a722f7 - Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents: 35
diff changeset
45 // Normal operators
35
c470b9356e35 - Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents: 32
diff changeset
46 Assign, Equal, NotEqual, Not,
38
640c45aaaaee - Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents: 36
diff changeset
47 LessEqual, Less,
640c45aaaaee - Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents: 36
diff changeset
48 GreaterEqual, Greater,
36
3c7210a722f7 - Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents: 35
diff changeset
49 LShiftAssign, LShift,
38
640c45aaaaee - Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents: 36
diff changeset
50 RShiftAssign,RShift,
640c45aaaaee - Added code for parsing GreaterEqual, Greater, RShiftAssign, RShift, URShiftAssign and URShift tokens.
aziz
parents: 36
diff changeset
51 URShiftAssign, URShift,
23
1a7903701a3d - Added code for parsing OrAssign, OrLogical and OrBinary tokens.
aziz
parents: 22
diff changeset
52 OrAssign, OrLogical, OrBinary,
24
903f91163f23 - Added code for parsing AndAssign, AndLogical and AndBinary tokens.
aziz
parents: 23
diff changeset
53 AndAssign, AndLogical, AndBinary,
25
9c866208b3f6 - Added code for parsing PlusAssign, PlusPlus, Plus, MinusAssign, MinusMinus, Minus tokens.
aziz
parents: 24
diff changeset
54 PlusAssign, PlusPlus, Plus,
9c866208b3f6 - Added code for parsing PlusAssign, PlusPlus, Plus, MinusAssign, MinusMinus, Minus tokens.
aziz
parents: 24
diff changeset
55 MinusAssign, MinusMinus, Minus,
32
d7011daa4740 - Added missing commas after the items in the messages table.
aziz
parents: 31
diff changeset
56 DivAssign, Div,
29
ef83eea26bbd - Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents: 28
diff changeset
57 MulAssign, Mul,
ef83eea26bbd - Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents: 28
diff changeset
58 ModAssign, Mod,
ef83eea26bbd - Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents: 28
diff changeset
59 XorAssign, Xor,
27
43b6bf56f0e9 - Added code for parsing CatAssign and Tilde tokens.
aziz
parents: 26
diff changeset
60 CatAssign, Catenate,
39
69b940398d7b - Added unittest to test correct parsing of operator tokens.
aziz
parents: 38
diff changeset
61 Tilde,
23
1a7903701a3d - Added code for parsing OrAssign, OrLogical and OrBinary tokens.
aziz
parents: 22
diff changeset
62
21
c785c122e4e6 - Added code for parsing Colon, Semicolon, Question, Comma and Dollar.
aziz
parents: 20
diff changeset
63 Colon,
c785c122e4e6 - Added code for parsing Colon, Semicolon, Question, Comma and Dollar.
aziz
parents: 20
diff changeset
64 Semicolon,
c785c122e4e6 - Added code for parsing Colon, Semicolon, Question, Comma and Dollar.
aziz
parents: 20
diff changeset
65 Question,
c785c122e4e6 - Added code for parsing Colon, Semicolon, Question, Comma and Dollar.
aziz
parents: 20
diff changeset
66 Comma,
c785c122e4e6 - Added code for parsing Colon, Semicolon, Question, Comma and Dollar.
aziz
parents: 20
diff changeset
67 Dollar,
c785c122e4e6 - Added code for parsing Colon, Semicolon, Question, Comma and Dollar.
aziz
parents: 20
diff changeset
68
28
3a9daccf7d96 - Added table for identifiers to Lexer.
aziz
parents: 27
diff changeset
69 /* Keywords:
3a9daccf7d96 - Added table for identifiers to Lexer.
aziz
parents: 27
diff changeset
70 NB.: Token.isKeyword() depends on this list being contiguous.
3a9daccf7d96 - Added table for identifiers to Lexer.
aziz
parents: 27
diff changeset
71 */
3a9daccf7d96 - Added table for identifiers to Lexer.
aziz
parents: 27
diff changeset
72 Abstract,Alias,Align,Asm,Assert,Auto,Body,
3a9daccf7d96 - Added table for identifiers to Lexer.
aziz
parents: 27
diff changeset
73 Bool,Break,Byte,Case,Cast,Catch,Cdouble,
3a9daccf7d96 - Added table for identifiers to Lexer.
aziz
parents: 27
diff changeset
74 Cent,Cfloat,Char,Class,Const,Continue,Creal,
3a9daccf7d96 - Added table for identifiers to Lexer.
aziz
parents: 27
diff changeset
75 Dchar,Debug,Default,Delegate,Delete,Deprecated,Do,
3a9daccf7d96 - Added table for identifiers to Lexer.
aziz
parents: 27
diff changeset
76 Double,Else,Enum,Export,Extern,False,Final,
3a9daccf7d96 - Added table for identifiers to Lexer.
aziz
parents: 27
diff changeset
77 Finally,Float,For,Foreach,Foreach_reverse,Function,Goto,
3a9daccf7d96 - Added table for identifiers to Lexer.
aziz
parents: 27
diff changeset
78 Idouble,If,Ifloat,Import,In,Inout,Int,
3a9daccf7d96 - Added table for identifiers to Lexer.
aziz
parents: 27
diff changeset
79 Interface,Invariant,Ireal,Is,Lazy,Long,Macro,
3a9daccf7d96 - Added table for identifiers to Lexer.
aziz
parents: 27
diff changeset
80 Mixin,Module,New,Null,Out,Override,Package,
3a9daccf7d96 - Added table for identifiers to Lexer.
aziz
parents: 27
diff changeset
81 Pragma,Private,Protected,Public,Real,Ref,Return,
3a9daccf7d96 - Added table for identifiers to Lexer.
aziz
parents: 27
diff changeset
82 Scope,Short,Static,Struct,Super,Switch,Synchronized,
3a9daccf7d96 - Added table for identifiers to Lexer.
aziz
parents: 27
diff changeset
83 Template,This,Throw,True,Try,Typedef,Typeid,
3a9daccf7d96 - Added table for identifiers to Lexer.
aziz
parents: 27
diff changeset
84 Typeof,Ubyte,Ucent,Uint,Ulong,Union,Unittest,
3a9daccf7d96 - Added table for identifiers to Lexer.
aziz
parents: 27
diff changeset
85 Ushort,Version,Void,Volatile,Wchar,While,With,
3a9daccf7d96 - Added table for identifiers to Lexer.
aziz
parents: 27
diff changeset
86
4
92df59b1ec4a - Started implementation of scan().
aziz
parents: 3
diff changeset
87 EOF
3
4bbce78bfb1e - Added TOK enum.
aziz
parents: 0
diff changeset
88 }
4bbce78bfb1e - Added TOK enum.
aziz
parents: 0
diff changeset
89
28
3a9daccf7d96 - Added table for identifiers to Lexer.
aziz
parents: 27
diff changeset
90 alias TOK.Abstract KeywordsBegin;
3a9daccf7d96 - Added table for identifiers to Lexer.
aziz
parents: 27
diff changeset
91 alias TOK.With KeywordsEnd;
3a9daccf7d96 - Added table for identifiers to Lexer.
aziz
parents: 27
diff changeset
92
0
8ba2570de175 Initial import.
aziz
parents:
diff changeset
93 struct Token
8ba2570de175 Initial import.
aziz
parents:
diff changeset
94 {
3
4bbce78bfb1e - Added TOK enum.
aziz
parents: 0
diff changeset
95 TOK type;
0
8ba2570de175 Initial import.
aziz
parents:
diff changeset
96 Position pos;
8ba2570de175 Initial import.
aziz
parents:
diff changeset
97
4
92df59b1ec4a - Started implementation of scan().
aziz
parents: 3
diff changeset
98 char* start;
92df59b1ec4a - Started implementation of scan().
aziz
parents: 3
diff changeset
99 char* end;
92df59b1ec4a - Started implementation of scan().
aziz
parents: 3
diff changeset
100
0
8ba2570de175 Initial import.
aziz
parents:
diff changeset
101 union
8ba2570de175 Initial import.
aziz
parents:
diff changeset
102 {
31
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix.
aziz
parents: 30
diff changeset
103 struct
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix.
aziz
parents: 30
diff changeset
104 {
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix.
aziz
parents: 30
diff changeset
105 string str;
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix.
aziz
parents: 30
diff changeset
106 char pf;
94f09f4e988e - Added struct for strings to Token with 'pf' = postfix.
aziz
parents: 30
diff changeset
107 }
46
e0dbd4722f74 - Scanning character literals correctly now.
aziz
parents: 39
diff changeset
108 dchar dchar_;
56
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 46
diff changeset
109 long long_;
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 46
diff changeset
110 ulong ulong_;
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 46
diff changeset
111 int int_;
63af7ddf52e1 - Started properly implementing number scanner. Added stub for real numbers.
aziz
parents: 46
diff changeset
112 uint uint_;
0
8ba2570de175 Initial import.
aziz
parents:
diff changeset
113 float f;
8ba2570de175 Initial import.
aziz
parents:
diff changeset
114 double d;
8ba2570de175 Initial import.
aziz
parents:
diff changeset
115 }
28
3a9daccf7d96 - Added table for identifiers to Lexer.
aziz
parents: 27
diff changeset
116
3a9daccf7d96 - Added table for identifiers to Lexer.
aziz
parents: 27
diff changeset
117 string span()
3a9daccf7d96 - Added table for identifiers to Lexer.
aziz
parents: 27
diff changeset
118 {
30
426767b94635 - Added code for parsing the '#line' special token.
aziz
parents: 29
diff changeset
119 assert(start && end);
28
3a9daccf7d96 - Added table for identifiers to Lexer.
aziz
parents: 27
diff changeset
120 return start[0 .. end - start];
3a9daccf7d96 - Added table for identifiers to Lexer.
aziz
parents: 27
diff changeset
121 }
3a9daccf7d96 - Added table for identifiers to Lexer.
aziz
parents: 27
diff changeset
122
3a9daccf7d96 - Added table for identifiers to Lexer.
aziz
parents: 27
diff changeset
123 bool isKeyword()
3a9daccf7d96 - Added table for identifiers to Lexer.
aziz
parents: 27
diff changeset
124 {
3a9daccf7d96 - Added table for identifiers to Lexer.
aziz
parents: 27
diff changeset
125 if (KeywordsBegin <= type && type <= KeywordsEnd)
3a9daccf7d96 - Added table for identifiers to Lexer.
aziz
parents: 27
diff changeset
126 return true;
3a9daccf7d96 - Added table for identifiers to Lexer.
aziz
parents: 27
diff changeset
127 return false;
3a9daccf7d96 - Added table for identifiers to Lexer.
aziz
parents: 27
diff changeset
128 }
0
8ba2570de175 Initial import.
aziz
parents:
diff changeset
129 }