|
1
|
1 module lexer.Token;
|
|
|
2
|
|
|
3 public
|
|
|
4 import misc.Location;
|
|
|
5
|
|
|
6 import Integer = tango.text.convert.Integer;
|
|
|
7
|
|
|
8 struct Token
|
|
|
9 {
|
|
|
10 Tok type;
|
|
|
11 Location location;
|
|
|
12 uint length;
|
|
|
13
|
|
|
14 static Token opCall (Tok type, Location location, uint length)
|
|
|
15 {
|
|
|
16 Token t;
|
|
|
17 t.type = type;
|
|
|
18 t.location = location;
|
|
|
19 t.length = length;
|
|
|
20 return t;
|
|
|
21 }
|
|
|
22
|
|
|
23 char[] getType ()
|
|
|
24 {
|
|
|
25 return typeToString[this.type];
|
|
|
26 }
|
|
|
27
|
|
|
28 char[] toString ()
|
|
|
29 {
|
|
|
30 return this.getType()~": Len: "~Integer.toString(this.length)
|
|
|
31 ~", Loc: "~location.toString;
|
|
|
32 }
|
|
|
33
|
|
|
34 char[] get ()
|
|
|
35 {
|
|
|
36 return location.get(length);
|
|
|
37 }
|
|
|
38 }
|
|
|
39
|
|
|
40 enum Tok : ushort
|
|
|
41 {
|
|
|
42 /* Non-code related tokens */
|
|
|
43 EOF,
|
|
|
44
|
|
|
45 /* Basic types */
|
|
|
46 Identifier,
|
|
|
47 Integer,
|
|
|
48
|
|
|
49 /* Basic operators */
|
|
|
50 Assign,
|
|
|
51 Add, Sub,
|
|
|
52 Mul, Div,
|
|
|
53 Comma,
|
|
|
54
|
|
|
55 /* Symbols */
|
|
|
56 OpenParentheses,
|
|
|
57 CloseParentheses,
|
|
|
58 OpenBrace,
|
|
|
59 CloseBrace,
|
|
|
60 Seperator,
|
|
|
61
|
|
|
62 /* Keywords */
|
|
|
63 Byte, Ubyte,
|
|
|
64 Short, Ushort,
|
|
|
65 Int, Uint,
|
|
|
66 Long, Ulong,
|
|
|
67
|
|
|
68 Float, Double,
|
|
|
69
|
|
|
70 Return,
|
|
|
71
|
|
|
72 }
|
|
|
73
|
|
|
74 public char[][Tok] typeToString;
|
|
|
75
|
|
|
76 static this()
|
|
|
77 {
|
|
|
78 typeToString =
|
|
|
79 [
|
|
|
80 Tok.EOF:"EOF"[],
|
|
|
81 Tok.Identifier:"Identifier",
|
|
|
82 Tok.Byte:"Byte",
|
|
|
83 Tok.Short:"Short",
|
|
|
84 Tok.Int:"Int",
|
|
|
85 Tok.Long:"Long",
|
|
|
86 Tok.OpenParentheses:"OpenParentheses",
|
|
|
87 Tok.CloseParentheses:"CloseParentheses",
|
|
|
88 Tok.OpenBrace:"OpenBrace",
|
|
|
89 Tok.CloseBrace:"CloseBrace",
|
|
|
90 Tok.Assign:"Assign",
|
|
|
91 Tok.Add:"Add",
|
|
|
92 Tok.Sub:"Sub",
|
|
|
93 Tok.Mul:"Mul",
|
|
|
94 Tok.Div:"Div",
|
|
|
95 Tok.Integer:"Interger",
|
|
|
96 Tok.Seperator:"Seperator"
|
|
|
97 ];
|
|
|
98 }
|
