Mercurial > projects > dil
annotate trunk/src/Token.d @ 36:3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
- Added comments to case statements.
author | aziz |
---|---|
date | Tue, 26 Jun 2007 08:36:00 +0000 |
parents | c470b9356e35 |
children | 640c45aaaaee |
rev | line source |
---|---|
0 | 1 /++ |
2 Author: Aziz Köksal | |
3 License: GPL2 | |
4 +/ | |
5 module Token; | |
6 | |
7 struct Position | |
8 { | |
9 size_t loc; | |
10 size_t col; | |
11 } | |
12 | |
3 | 13 enum TOK |
14 { | |
15 Identifier, | |
4 | 16 Comment, |
9
5d6968cc751e
- Parsing string and character literals now (rudimentary implementation.)
aziz
parents:
4
diff
changeset
|
17 String, |
5d6968cc751e
- Parsing string and character literals now (rudimentary implementation.)
aziz
parents:
4
diff
changeset
|
18 Character, |
15 | 19 Number, |
36
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
20 |
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
21 // Brackets |
20 | 22 LParen, |
23 RParen, | |
24 LBracket, | |
25 RBracket, | |
26 LBrace, | |
27 RBrace, | |
21
c785c122e4e6
- Added code for parsing Colon, Semicolon, Question, Comma and Dollar.
aziz
parents:
20
diff
changeset
|
28 |
22
b05fff8e2ce4
- Added code for parsing Dot, Slice and Ellipses tokens.
aziz
parents:
21
diff
changeset
|
29 Dot, Slice, Ellipses, |
b05fff8e2ce4
- Added code for parsing Dot, Slice and Ellipses tokens.
aziz
parents:
21
diff
changeset
|
30 |
36
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
31 // Floating point operators |
35
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
32
diff
changeset
|
32 Unordered, |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
32
diff
changeset
|
33 UorE, |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
32
diff
changeset
|
34 UorG, |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
32
diff
changeset
|
35 UorGorE, |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
32
diff
changeset
|
36 UorL, |
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
32
diff
changeset
|
37 UorLorE, |
36
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
38 LorEorG, |
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
39 LorG, |
35
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
32
diff
changeset
|
40 |
36
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
41 // Normal operators |
35
c470b9356e35
- Added code for parsing Unordered, UorE, UorG, UorGorE, UorL, UorLorE, NotEqual and Not tokens.
aziz
parents:
32
diff
changeset
|
42 Assign, Equal, NotEqual, Not, |
36
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
43 LessEqual, LessThan, |
3c7210a722f7
- Added code for parsing LorEorG, LorG, LessEqual, LessThan, LShiftAssign and LShift tokens.
aziz
parents:
35
diff
changeset
|
44 LShiftAssign, LShift, |
23
1a7903701a3d
- Added code for parsing OrAssign, OrLogical and OrBinary tokens.
aziz
parents:
22
diff
changeset
|
45 OrAssign, OrLogical, OrBinary, |
24
903f91163f23
- Added code for parsing AndAssign, AndLogical and AndBinary tokens.
aziz
parents:
23
diff
changeset
|
46 AndAssign, AndLogical, AndBinary, |
25
9c866208b3f6
- Added code for parsing PlusAssign, PlusPlus, Plus, MinusAssign, MinusMinus, Minus tokens.
aziz
parents:
24
diff
changeset
|
47 PlusAssign, PlusPlus, Plus, |
9c866208b3f6
- Added code for parsing PlusAssign, PlusPlus, Plus, MinusAssign, MinusMinus, Minus tokens.
aziz
parents:
24
diff
changeset
|
48 MinusAssign, MinusMinus, Minus, |
32
d7011daa4740
- Added missing commas after the items in the messages table.
aziz
parents:
31
diff
changeset
|
49 DivAssign, Div, |
29
ef83eea26bbd
- Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents:
28
diff
changeset
|
50 MulAssign, Mul, |
ef83eea26bbd
- Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents:
28
diff
changeset
|
51 ModAssign, Mod, |
ef83eea26bbd
- Added code for parsing MulAssign, Mul, ModAssign, Mod, XorAssign and Xor tokens.
aziz
parents:
28
diff
changeset
|
52 XorAssign, Xor, |
27 | 53 CatAssign, Catenate, |
23
1a7903701a3d
- Added code for parsing OrAssign, OrLogical and OrBinary tokens.
aziz
parents:
22
diff
changeset
|
54 |
27 | 55 Tilde, |
21
c785c122e4e6
- Added code for parsing Colon, Semicolon, Question, Comma and Dollar.
aziz
parents:
20
diff
changeset
|
56 Colon, |
c785c122e4e6
- Added code for parsing Colon, Semicolon, Question, Comma and Dollar.
aziz
parents:
20
diff
changeset
|
57 Semicolon, |
c785c122e4e6
- Added code for parsing Colon, Semicolon, Question, Comma and Dollar.
aziz
parents:
20
diff
changeset
|
58 Question, |
c785c122e4e6
- Added code for parsing Colon, Semicolon, Question, Comma and Dollar.
aziz
parents:
20
diff
changeset
|
59 Comma, |
c785c122e4e6
- Added code for parsing Colon, Semicolon, Question, Comma and Dollar.
aziz
parents:
20
diff
changeset
|
60 Dollar, |
c785c122e4e6
- Added code for parsing Colon, Semicolon, Question, Comma and Dollar.
aziz
parents:
20
diff
changeset
|
61 |
28 | 62 /* Keywords: |
63 NB.: Token.isKeyword() depends on this list being contiguous. | |
64 */ | |
65 Abstract,Alias,Align,Asm,Assert,Auto,Body, | |
66 Bool,Break,Byte,Case,Cast,Catch,Cdouble, | |
67 Cent,Cfloat,Char,Class,Const,Continue,Creal, | |
68 Dchar,Debug,Default,Delegate,Delete,Deprecated,Do, | |
69 Double,Else,Enum,Export,Extern,False,Final, | |
70 Finally,Float,For,Foreach,Foreach_reverse,Function,Goto, | |
71 Idouble,If,Ifloat,Import,In,Inout,Int, | |
72 Interface,Invariant,Ireal,Is,Lazy,Long,Macro, | |
73 Mixin,Module,New,Null,Out,Override,Package, | |
74 Pragma,Private,Protected,Public,Real,Ref,Return, | |
75 Scope,Short,Static,Struct,Super,Switch,Synchronized, | |
76 Template,This,Throw,True,Try,Typedef,Typeid, | |
77 Typeof,Ubyte,Ucent,Uint,Ulong,Union,Unittest, | |
78 Ushort,Version,Void,Volatile,Wchar,While,With, | |
79 | |
4 | 80 EOF |
3 | 81 } |
82 | |
28 | 83 alias TOK.Abstract KeywordsBegin; |
84 alias TOK.With KeywordsEnd; | |
85 | |
0 | 86 struct Token |
87 { | |
3 | 88 TOK type; |
0 | 89 Position pos; |
90 | |
4 | 91 char* start; |
92 char* end; | |
93 | |
0 | 94 union |
95 { | |
31
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
96 struct |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
97 { |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
98 string str; |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
99 char pf; |
94f09f4e988e
- Added struct for strings to Token with 'pf' = postfix.
aziz
parents:
30
diff
changeset
|
100 } |
18
c48d2125f1e2
- Moved code for scanning character literals to separate function.
aziz
parents:
17
diff
changeset
|
101 dchar chr; |
30 | 102 uint _uint; |
0 | 103 float f; |
104 double d; | |
105 } | |
28 | 106 |
107 string span() | |
108 { | |
30 | 109 assert(start && end); |
28 | 110 return start[0 .. end - start]; |
111 } | |
112 | |
113 bool isKeyword() | |
114 { | |
115 if (KeywordsBegin <= type && type <= KeywordsEnd) | |
116 return true; | |
117 return false; | |
118 } | |
0 | 119 } |